





from dataclasses import dataclass, field
from typing import Tuple

from ..utils import cached_property, is_tf_available, logging, requires_backends
from .benchmark_args_utils import BenchmarkArguments


if is_tf_available():
		import tensorflow as tf


lowercase_		:				Union[str, Any]				=						logging.get_logger(__name__)


@dataclass
class 			lowercase				(							__snake_case     ):



						"""simple docstring"""

						_UpperCamelCase  :	Any								=	[
						    "no_inference",
						    "no_cuda",
						    "no_tpu",
						    "no_speed",
						    "no_memory",
						    "no_env_print",
						    "no_multi_process",
						]







						def __init__(    self			:		int ,				**lowerCamelCase_			:		Tuple					):





										'''simple docstring'''







										for deprecated_arg in self.deprecated_args:
														if deprecated_arg in kwargs:
																		_snake_case :			Optional[Any]  						=   deprecated_arg[3:]
																		_snake_case :			Optional[int]  						=   not kwargs.pop(_lowercase					)
																		logger.warning(
																		    f'''{deprecated_arg} is depreciated. Please use --no-{positive_arg} or'''
																		    f''' {positive_arg}={kwargs[positive_arg]}'''					)
										_snake_case :			Union[str, Any]  						=   kwargs.pop('tpu_name' ,				self.tpu_name					)
										_snake_case :			List[Any]  						=   kwargs.pop('device_idx' ,				self.device_idx					)
										_snake_case :			Optional[Any]  						=   kwargs.pop('eager_mode' ,				self.eager_mode					)
										_snake_case :			Tuple  						=   kwargs.pop('use_xla' ,				self.use_xla					)
										super().__init__(**_lowercase					)

						_UpperCamelCase  :	Union[str, Any]								=	field(
						    default=__snake_case		,			metadata={"help": "Name of TPU"}		,			)
						_UpperCamelCase  :	Optional[Any]								=	field(
						    default=0		,			metadata={"help": "CPU / GPU device index. Defaults to 0."}		,			)
						_UpperCamelCase  :	Optional[Any]								=	field(default=__snake_case		,			metadata={"help": "Benchmark models in eager model."}     )
						_UpperCamelCase  :	str								=	field(
						    default=__snake_case		,			metadata={
						        "help": "Benchmark models using XLA JIT compilation. Note that `eager_model` has to be set to `False`."
						    }		,			)







						@cached_property
						def 		__UpperCAmelCase       (    self			:		Any					):





										'''simple docstring'''







										requires_backends(self ,				['tf']					)
										_snake_case :			List[Any]  						=   None
										if self.tpu:
														try:
																		if self.tpu_name:
																						_snake_case :			List[str]  						=   tf.distribute.cluster_resolver.TPUClusterResolver(self.tpu_name					)
																		else:
																						_snake_case :			List[Any]  						=   tf.distribute.cluster_resolver.TPUClusterResolver()
														except ValueError:
																		_snake_case :			List[Any]  						=   None
										return tpu







						@cached_property
						def 		__UpperCAmelCase       (    self			:		List[str]					):





										'''simple docstring'''







										requires_backends(self ,				['tf']					)
										if self.is_tpu:
														tf.config.experimental_connect_to_cluster(self._setup_tpu					)
														tf.tpu.experimental.initialize_tpu_system(self._setup_tpu					)

														_snake_case :			Optional[Any]  						=   tf.distribute.TPUStrategy(self._setup_tpu					)
										else:
														# currently no multi gpu is allowed
														if self.is_gpu:
																		# TODO: Currently only single GPU is supported
																		tf.config.set_visible_devices(self.gpu_list[self.device_idx] ,				'GPU'					)
																		_snake_case :			Union[str, Any]  						=   tf.distribute.OneDeviceStrategy(device=f'''/gpu:{self.device_idx}'''					)
														else:
																		tf.config.set_visible_devices([] ,				'GPU'					)  # disable GPU
																		_snake_case :			Optional[Any]  						=   tf.distribute.OneDeviceStrategy(device=f'''/cpu:{self.device_idx}'''					)

										return strategy







						@property
						def 		__UpperCAmelCase       (    self			:		Tuple					):





										'''simple docstring'''







										requires_backends(self ,				['tf']					)
										return self._setup_tpu is not None







						@property
						def 		__UpperCAmelCase       (    self			:		Dict					):





										'''simple docstring'''







										requires_backends(self ,				['tf']					)
										return self._setup_strategy







						@property
						def 		__UpperCAmelCase       (    self			:		int					):





										'''simple docstring'''







										requires_backends(self ,				['tf']					)
										return tf.config.list_physical_devices('GPU'					)







						@property
						def 		__UpperCAmelCase       (    self			:		Dict					):





										'''simple docstring'''







										requires_backends(self ,				['tf']					)
										if self.cuda:
														return len(self.gpu_list					)
										return 0







						@property
						def 		__UpperCAmelCase       (    self			:		int					):





										'''simple docstring'''







										return self.n_gpu > 0












import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
lowercase_		:				Optional[int]				=						object()

# For specifying empty leaf dict `{}`
lowercase_		:				List[Any]				=						object()

def       A__(		__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			Optional[Any]  						=   tuple((re.compile(x + '$' ) for x in qs) )
				for i in range(len(__lowerCAmelCase ) - len(__lowerCAmelCase ) + 1 ):
								_snake_case :			Tuple  						=   [x.match(__lowerCAmelCase ) for x, y in zip(__lowerCAmelCase	,			ks[i:] )]
								if matches and all(__lowerCAmelCase ):
												return True
				return False

def       A__(		__lowerCAmelCase ):
				def replace(__lowerCAmelCase	,			__lowerCAmelCase ):
								for rule, replacement in rules:
												if _match(__lowerCAmelCase	,			__lowerCAmelCase ):
																return replacement
								return val

				return replace

def       A__(		):
				return [
				    # embeddings
				    (("transformer", "wpe", "embedding"), P('mp'	,			__lowerCAmelCase )),
				    (("transformer", "wte", "embedding"), P('mp'	,			__lowerCAmelCase )),
				    # atention
				    (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(__lowerCAmelCase	,			'mp' )),
				    (("attention", "out_proj", "kernel"), P('mp'	,			__lowerCAmelCase )),
				    (("attention", "out_proj", "bias"), None),
				    # mlp
				    (("mlp", "c_fc", "kernel"), P(__lowerCAmelCase	,			'mp' )),
				    (("mlp", "c_fc", "bias"), P('mp' )),
				    (("mlp", "c_proj", "kernel"), P('mp'	,			__lowerCAmelCase )),
				    (("mlp", "c_proj", "bias"), None),
				    # layer norms
				    ((r"ln_\d+", "bias"), None),
				    ((r"\d+", r"ln_\d+", "scale"), None),
				    (("ln_f", "bias"), None),
				    (("ln_f", "scale"), None),
				]

def       A__(		__lowerCAmelCase ):
				_snake_case :			Optional[Any]  						=   _get_partition_rules()
				_snake_case :			Optional[Any]  						=   _replacement_rules(__lowerCAmelCase )
				_snake_case :			str  						=   {k: _unmatched for k in flatten_dict(__lowerCAmelCase )}
				_snake_case :			str  						=   {k: replace(__lowerCAmelCase	,			__lowerCAmelCase ) for k, v in initd.items()}
				assert _unmatched not in result.values(), "Incomplete partition spec."
				return freeze(unflatten_dict(__lowerCAmelCase ) )






