
"""simple docstring"""




from typing import Optional, Tuple, Union

import tensorflow as tf

from ...activations_tf import ACTaFN
from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
from ...modeling_tf_outputs import (
    TFBaseModelOutputWithNoAttention,
    TFBaseModelOutputWithPoolingAndNoAttention,
    TFSequenceClassifierOutput,
)
from ...modeling_tf_utils import TFPreTrainedModel, TFSequenceClassificationLoss, keras_serializable, unpack_inputs
from ...tf_utils import shape_list
from ...utils import logging
from .configuration_regnet import RegNetConfig


A__      :	Optional[int]          =    logging.get_logger(__name__)

# General docstring
A__      :	Tuple          =    '''RegNetConfig'''

# Base docstring
A__      :	Any          =    '''facebook/regnet-y-040'''
A__      :	int          =    [1, 1_088, 7, 7]

# Image classification docstring
A__      :	Any          =    '''facebook/regnet-y-040'''
A__      :	Union[str, Any]          =    '''tabby, tabby cat'''

A__      :	Optional[Any]          =    [
    '''facebook/regnet-y-040''',
    # See all regnet models at https://huggingface.co/models?filter=regnet
]







class   _lowercase						(       tf.keras.layers.Layer				):


     '''simple docstring'''





     def __init__(      self ,			__UpperCamelCase ,			__UpperCamelCase = 3 ,			__UpperCamelCase = 1 ,			__UpperCamelCase = 1 ,			__UpperCamelCase = "relu" ,			**__UpperCamelCase ,			)->					List[Any]:
          super().__init__(**lowerCAmelCase__  )
          # The padding and conv has been verified in
          # https://colab.research.google.com/gist/sayakpaul/854bc10eeaf21c9ee2119e0b9f3841a7/scratchpad.ipynb
          UpperCAmelCase__							: Optional[int] 							=						tf.keras.layers.ZeroPaddingaD(padding=kernel_size // 2  )
          UpperCAmelCase__							: Optional[Any] 							=						tf.keras.layers.ConvaD(
              filters=lowerCAmelCase__ ,			kernel_size=lowerCAmelCase__ ,			strides=lowerCAmelCase__ ,			padding="VALID" ,			groups=lowerCAmelCase__ ,			use_bias=lowerCAmelCase__ ,			name="convolution" ,			)
          UpperCAmelCase__							: str 							=						tf.keras.layers.BatchNormalization(epsilon=1E-5 ,			momentum=0.9 ,			name="normalization"  )
          UpperCAmelCase__							: Dict 							=						ACTaFN[activation] if activation is not None else tf.identity



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Optional[int]:
          UpperCAmelCase__							: Any 							=						self.convolution(self.padding(lowerCAmelCase__  )  )
          UpperCAmelCase__							: Optional[int] 							=						self.normalization(lowerCAmelCase__  )
          UpperCAmelCase__							: List[str] 							=						self.activation(lowerCAmelCase__  )
          return hidden_state







class   _lowercase						(       tf.keras.layers.Layer				):


     '''simple docstring'''





     def __init__(      self ,			__UpperCamelCase ,			**__UpperCamelCase  )->					int:
          super().__init__(**lowerCAmelCase__  )
          UpperCAmelCase__							: Tuple 							=						config.num_channels
          UpperCAmelCase__							: Optional[Any] 							=						TFRegNetConvLayer(
              out_channels=config.embedding_size ,			kernel_size=3 ,			stride=2 ,			activation=config.hidden_act ,			name="embedder" ,			)



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Optional[int]:
          UpperCAmelCase__							: Union[str, Any] 							=						shape_list(lowerCAmelCase__  )[1]
          if tf.executing_eagerly() and num_channels != self.num_channels:
               raise ValueError(
                   "Make sure that the channel dimension of the pixel values match with the one set in the configuration."  )

          # When running on CPU, `tf.keras.layers.Conv2D` doesn't support `NCHW` format.
          # So change the input format from `NCHW` to `NHWC`.
          # shape = (batch_size, in_height, in_width, in_channels=num_channels)
          UpperCAmelCase__							: List[str] 							=						tf.transpose(lowerCAmelCase__ ,			perm=(0, 2, 3, 1)  )
          UpperCAmelCase__							: Optional[Any] 							=						self.embedder(lowerCAmelCase__  )
          return hidden_state







class   _lowercase						(       tf.keras.layers.Layer				):


     '''simple docstring'''





     def __init__(      self ,			__UpperCamelCase ,			__UpperCamelCase = 2 ,			**__UpperCamelCase  )->					List[str]:
          super().__init__(**lowerCAmelCase__  )
          UpperCAmelCase__							: Union[str, Any] 							=						tf.keras.layers.ConvaD(
              filters=lowerCAmelCase__ ,			kernel_size=1 ,			strides=lowerCAmelCase__ ,			use_bias=lowerCAmelCase__ ,			name="convolution"  )
          UpperCAmelCase__							: Dict 							=						tf.keras.layers.BatchNormalization(epsilon=1E-5 ,			momentum=0.9 ,			name="normalization"  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase = False  )->					tf.Tensor:
          return self.normalization(self.convolution(lowerCAmelCase__  ) ,			training=lowerCAmelCase__  )







class   _lowercase						(       tf.keras.layers.Layer				):


     '''simple docstring'''





     def __init__(      self ,			__UpperCamelCase ,			__UpperCamelCase ,			**__UpperCamelCase  )->					Optional[int]:
          super().__init__(**lowerCAmelCase__  )
          UpperCAmelCase__							: str 							=						tf.keras.layers.GlobalAveragePoolingaD(keepdims=lowerCAmelCase__ ,			name="pooler"  )
          UpperCAmelCase__							: Optional[int] 							=						[
              tf.keras.layers.ConvaD(filters=lowerCAmelCase__ ,			kernel_size=1 ,			activation="relu" ,			name="attention.0"  ),
              tf.keras.layers.ConvaD(filters=lowerCAmelCase__ ,			kernel_size=1 ,			activation="sigmoid" ,			name="attention.2"  ),
          ]



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Union[str, Any]:
          # [batch_size, h, w, num_channels] -> [batch_size, 1, 1, num_channels]
          UpperCAmelCase__							: Dict 							=						self.pooler(lowerCAmelCase__  )
          for layer_module in self.attention:
               UpperCAmelCase__							: Optional[Any] 							=						layer_module(lowerCAmelCase__  )
          UpperCAmelCase__							: int 							=						hidden_state * pooled
          return hidden_state







class   _lowercase						(       tf.keras.layers.Layer				):


     '''simple docstring'''





     def __init__(      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase = 1 ,			**__UpperCamelCase  )->					int:
          super().__init__(**lowerCAmelCase__  )
          UpperCAmelCase__							: Optional[int] 							=						in_channels != out_channels or stride != 1
          UpperCAmelCase__							: Dict 							=						max(1 ,			out_channels // config.groups_width  )
          UpperCAmelCase__							: Optional[Any] 							=						(
              TFRegNetShortCut(lowerCAmelCase__ ,			stride=lowerCAmelCase__ ,			name="shortcut"  )
              if should_apply_shortcut
              else tf.keras.layers.Activation("linear" ,			name="shortcut"  )
          )
          # `self.layers` instead of `self.layer` because that is a reserved argument.
          UpperCAmelCase__							: Dict 							=						[
              TFRegNetConvLayer(lowerCAmelCase__ ,			kernel_size=1 ,			activation=config.hidden_act ,			name="layer.0"  ),
              TFRegNetConvLayer(
                  lowerCAmelCase__ ,			stride=lowerCAmelCase__ ,			groups=lowerCAmelCase__ ,			activation=config.hidden_act ,			name="layer.1"  ),
              TFRegNetConvLayer(lowerCAmelCase__ ,			kernel_size=1 ,			activation=lowerCAmelCase__ ,			name="layer.2"  ),
          ]
          UpperCAmelCase__							: List[str] 							=						ACTaFN[config.hidden_act]



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Optional[Any]:
          UpperCAmelCase__							: Any 							=						hidden_state
          for layer_module in self.layers:
               UpperCAmelCase__							: Tuple 							=						layer_module(lowerCAmelCase__  )
          UpperCAmelCase__							: Dict 							=						self.shortcut(lowerCAmelCase__  )
          hidden_state += residual
          UpperCAmelCase__							: int 							=						self.activation(lowerCAmelCase__  )
          return hidden_state







class   _lowercase						(       tf.keras.layers.Layer				):


     '''simple docstring'''





     def __init__(      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase = 1 ,			**__UpperCamelCase  )->					List[Any]:
          super().__init__(**lowerCAmelCase__  )
          UpperCAmelCase__							: List[str] 							=						in_channels != out_channels or stride != 1
          UpperCAmelCase__							: Any 							=						max(1 ,			out_channels // config.groups_width  )
          UpperCAmelCase__							: str 							=						(
              TFRegNetShortCut(lowerCAmelCase__ ,			stride=lowerCAmelCase__ ,			name="shortcut"  )
              if should_apply_shortcut
              else tf.keras.layers.Activation("linear" ,			name="shortcut"  )
          )
          UpperCAmelCase__							: Optional[int] 							=						[
              TFRegNetConvLayer(lowerCAmelCase__ ,			kernel_size=1 ,			activation=config.hidden_act ,			name="layer.0"  ),
              TFRegNetConvLayer(
                  lowerCAmelCase__ ,			stride=lowerCAmelCase__ ,			groups=lowerCAmelCase__ ,			activation=config.hidden_act ,			name="layer.1"  ),
              TFRegNetSELayer(lowerCAmelCase__ ,			reduced_channels=int(round(in_channels / 4  )  ) ,			name="layer.2"  ),
              TFRegNetConvLayer(lowerCAmelCase__ ,			kernel_size=1 ,			activation=lowerCAmelCase__ ,			name="layer.3"  ),
          ]
          UpperCAmelCase__							: Union[str, Any] 							=						ACTaFN[config.hidden_act]



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Any:
          UpperCAmelCase__							: Union[str, Any] 							=						hidden_state
          for layer_module in self.layers:
               UpperCAmelCase__							: str 							=						layer_module(lowerCAmelCase__  )
          UpperCAmelCase__							: Union[str, Any] 							=						self.shortcut(lowerCAmelCase__  )
          hidden_state += residual
          UpperCAmelCase__							: Optional[int] 							=						self.activation(lowerCAmelCase__  )
          return hidden_state







class   _lowercase						(       tf.keras.layers.Layer				):


     '''simple docstring'''





     def __init__(      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase = 2 ,			__UpperCamelCase = 2 ,			**__UpperCamelCase  )->					List[Any]:
          super().__init__(**lowerCAmelCase__  )

          UpperCAmelCase__							: Tuple 							=						TFRegNetXLayer if config.layer_type == "x" else TFRegNetYLayer
          UpperCAmelCase__							: Optional[Any] 							=						[
              # downsampling is done in the first layer with stride of 2
              layer(lowerCAmelCase__ ,			lowerCAmelCase__ ,			lowerCAmelCase__ ,			stride=lowerCAmelCase__ ,			name="layers.0"  ),
              *[layer(lowerCAmelCase__ ,			lowerCAmelCase__ ,			lowerCAmelCase__ ,			name=F"layers.{i+1}"  ) for i in range(depth - 1  )],
          ]



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					int:
          for layer_module in self.layers:
               UpperCAmelCase__							: Dict 							=						layer_module(lowerCAmelCase__  )
          return hidden_state







class   _lowercase						(       tf.keras.layers.Layer				):


     '''simple docstring'''





     def __init__(      self ,			__UpperCamelCase ,			**__UpperCamelCase  )->					int:
          super().__init__(**lowerCAmelCase__  )
          UpperCAmelCase__							: str 							=						[]
          # based on `downsample_in_first_stage`, the first layer of the first stage may or may not downsample the input
          self.stages.append(
              TFRegNetStage(
                  lowerCAmelCase__ ,			config.embedding_size ,			config.hidden_sizes[0] ,			stride=2 if config.downsample_in_first_stage else 1 ,			depth=config.depths[0] ,			name="stages.0" ,			)  )
          UpperCAmelCase__							: Optional[Any] 							=						zip(config.hidden_sizes ,			config.hidden_sizes[1:]  )
          for i, ((in_channels, out_channels), depth) in enumerate(zip(lowerCAmelCase__ ,			config.depths[1:]  )  ):
               self.stages.append(TFRegNetStage(lowerCAmelCase__ ,			lowerCAmelCase__ ,			lowerCAmelCase__ ,			depth=lowerCAmelCase__ ,			name=F"stages.{i+1}"  )  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase = False ,			__UpperCamelCase = True  )->					TFBaseModelOutputWithNoAttention:
          UpperCAmelCase__							: int 							=						() if output_hidden_states else None

          for stage_module in self.stages:
               if output_hidden_states:
                    UpperCAmelCase__							: Optional[int] 							=						hidden_states + (hidden_state,)

               UpperCAmelCase__							: Any 							=						stage_module(lowerCAmelCase__  )

          if output_hidden_states:
               UpperCAmelCase__							: Union[str, Any] 							=						hidden_states + (hidden_state,)

          if not return_dict:
               return tuple(v for v in [hidden_state, hidden_states] if v is not None  )

          return TFBaseModelOutputWithNoAttention(last_hidden_state=lowerCAmelCase__ ,			hidden_states=lowerCAmelCase__  )







@keras_serializable
class   _lowercase						(       tf.keras.layers.Layer				):


     '''simple docstring'''





     _A								=							RegNetConfig



     def __init__(      self ,			__UpperCamelCase ,			**__UpperCamelCase  )->					Union[str, Any]:
          super().__init__(**lowerCAmelCase__  )
          UpperCAmelCase__							: int 							=						config
          UpperCAmelCase__							: str 							=						TFRegNetEmbeddings(lowerCAmelCase__ ,			name="embedder"  )
          UpperCAmelCase__							: Union[str, Any] 							=						TFRegNetEncoder(lowerCAmelCase__ ,			name="encoder"  )
          UpperCAmelCase__							: Union[str, Any] 							=						tf.keras.layers.GlobalAveragePoolingaD(keepdims=lowerCAmelCase__ ,			name="pooler"  )



     @unpack_inputs
     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase = None ,			__UpperCamelCase = None ,			__UpperCamelCase = False ,			)->					TFBaseModelOutputWithPoolingAndNoAttention:
          UpperCAmelCase__							: Any 							=						(
              output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
          )
          UpperCAmelCase__							: int 							=						return_dict if return_dict is not None else self.config.use_return_dict

          UpperCAmelCase__							: Any 							=						self.embedder(lowerCAmelCase__ ,			training=lowerCAmelCase__  )

          UpperCAmelCase__							: Tuple 							=						self.encoder(
              lowerCAmelCase__ ,			output_hidden_states=lowerCAmelCase__ ,			return_dict=lowerCAmelCase__ ,			training=lowerCAmelCase__  )

          UpperCAmelCase__							: int 							=						encoder_outputs[0]
          UpperCAmelCase__							: List[Any] 							=						self.pooler(lowerCAmelCase__  )

          # Change to NCHW output format have uniformity in the modules
          UpperCAmelCase__							: Tuple 							=						tf.transpose(lowerCAmelCase__ ,			perm=(0, 3, 1, 2)  )
          UpperCAmelCase__							: Optional[Any] 							=						tf.transpose(lowerCAmelCase__ ,			perm=(0, 3, 1, 2)  )

          # Change the other hidden state outputs to NCHW as well
          if output_hidden_states:
               UpperCAmelCase__							: Tuple 							=						tuple([tf.transpose(lowerCAmelCase__ ,			perm=(0, 3, 1, 2)  ) for h in encoder_outputs[1]]  )

          if not return_dict:
               return (last_hidden_state, pooled_output) + encoder_outputs[1:]

          return TFBaseModelOutputWithPoolingAndNoAttention(
              last_hidden_state=lowerCAmelCase__ ,			pooler_output=lowerCAmelCase__ ,			hidden_states=hidden_states if output_hidden_states else encoder_outputs.hidden_states ,			)







class   _lowercase						(       a__				):


     '''simple docstring'''





     _A								=							RegNetConfig
     _A								=							"""regnet"""
     _A								=							"""pixel_values"""



     @property
     def  lowerCAmelCase__       (      self  )->					str:
          return {"pixel_values": tf.TensorSpec(shape=(None, self.config.num_channels, 2_24, 2_24) ,			dtype=tf.floataa  )}





A__      :	Optional[Any]          =    R'''
    Parameters:
    This model is a Tensorflow
    [tf.keras.layers.Layer](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer) sub-class. Use it as a
    regular Tensorflow Module and refer to the Tensorflow documentation for all matter related to general usage and
    behavior.
        config ([`RegNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
'''

A__      :	Dict          =    R'''
    Args:
        pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`ConveNextImageProcessor.__call__`] for details.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
'''







@add_start_docstrings(
    'The bare RegNet model outputting raw features without any specific head on top.'	,   a__	,   )
class   _lowercase						(       a__				):


     '''simple docstring'''





     def __init__(      self ,			__UpperCamelCase ,			*__UpperCamelCase ,			**__UpperCamelCase  )->					List[str]:
          super().__init__(lowerCAmelCase__ ,			*lowerCAmelCase__ ,			**lowerCAmelCase__  )
          UpperCAmelCase__							: Union[str, Any] 							=						TFRegNetMainLayer(lowerCAmelCase__ ,			name="regnet"  )



     @unpack_inputs
     @add_start_docstrings_to_model_forward(lowerCAmelCase__  )
     @add_code_sample_docstrings(
         checkpoint=_CHECKPOINT_FOR_DOC ,			output_type=lowerCAmelCase__ ,			config_class=_CONFIG_FOR_DOC ,			modality="vision" ,			expected_output=_EXPECTED_OUTPUT_SHAPE ,			)
     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase = None ,			__UpperCamelCase = None ,			__UpperCamelCase=False ,			)->					Union[TFBaseModelOutputWithPoolingAndNoAttention, Tuple[tf.Tensor]]:
          UpperCAmelCase__							: Optional[int] 							=						(
              output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
          )
          UpperCAmelCase__							: Union[str, Any] 							=						return_dict if return_dict is not None else self.config.use_return_dict

          UpperCAmelCase__							: List[str] 							=						self.regnet(
              pixel_values=lowerCAmelCase__ ,			output_hidden_states=lowerCAmelCase__ ,			return_dict=lowerCAmelCase__ ,			training=lowerCAmelCase__ ,			)
          if not return_dict:
               return (outputs[0],) + outputs[1:]

          return TFBaseModelOutputWithPoolingAndNoAttention(
              last_hidden_state=outputs.last_hidden_state ,			pooler_output=outputs.pooler_output ,			hidden_states=outputs.hidden_states ,			)







@add_start_docstrings(
    '\n    RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    '	,   a__	,   )
class   _lowercase						(       a__	,   a__				):


     '''simple docstring'''





     def __init__(      self ,			__UpperCamelCase ,			*__UpperCamelCase ,			**__UpperCamelCase  )->					str:
          super().__init__(lowerCAmelCase__ ,			*lowerCAmelCase__ ,			**lowerCAmelCase__  )
          UpperCAmelCase__							: Optional[Any] 							=						config.num_labels
          UpperCAmelCase__							: Any 							=						TFRegNetMainLayer(lowerCAmelCase__ ,			name="regnet"  )
          # classification head
          UpperCAmelCase__							: List[str] 							=						[
              tf.keras.layers.Flatten(),
              tf.keras.layers.Dense(config.num_labels ,			name="classifier.1"  ) if config.num_labels > 0 else tf.identity,
          ]



     @unpack_inputs
     @add_start_docstrings_to_model_forward(lowerCAmelCase__  )
     @add_code_sample_docstrings(
         checkpoint=_IMAGE_CLASS_CHECKPOINT ,			output_type=lowerCAmelCase__ ,			config_class=_CONFIG_FOR_DOC ,			expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT ,			)
     def  lowerCAmelCase__       (      self ,			__UpperCamelCase = None ,			__UpperCamelCase = None ,			__UpperCamelCase = None ,			__UpperCamelCase = None ,			__UpperCamelCase=False ,			)->					Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]:
          UpperCAmelCase__							: Any 							=						(
              output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
          )
          UpperCAmelCase__							: List[str] 							=						return_dict if return_dict is not None else self.config.use_return_dict

          UpperCAmelCase__							: Union[str, Any] 							=						self.regnet(
              lowerCAmelCase__ ,			output_hidden_states=lowerCAmelCase__ ,			return_dict=lowerCAmelCase__ ,			training=lowerCAmelCase__  )

          UpperCAmelCase__							: Dict 							=						outputs.pooler_output if return_dict else outputs[1]

          UpperCAmelCase__							: List[Any] 							=						self.classifier[0](lowerCAmelCase__  )
          UpperCAmelCase__							: Dict 							=						self.classifier[1](lowerCAmelCase__  )

          UpperCAmelCase__							: Tuple 							=						None if labels is None else self.hf_compute_loss(labels=lowerCAmelCase__ ,			logits=lowerCAmelCase__  )

          if not return_dict:
               UpperCAmelCase__							: str 							=						(logits,) + outputs[2:]
               return ((loss,) + output) if loss is not None else output

          return TFSequenceClassifierOutput(loss=lowerCAmelCase__ ,			logits=lowerCAmelCase__ ,			hidden_states=outputs.hidden_states  )





"""simple docstring"""




import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
    import tensorflow as tf
    from tensorflow.python.framework.errors_impl import ResourceExhaustedError

    from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
    import pyanvml.pyanvml as nvml

A__      :	Optional[Any]          =    logging.get_logger(__name__)



def 							a__ (				lowerCAmelCase	:       bool       ,    lowerCAmelCase	:       bool		):






     '''simple docstring'''
     def run_func(lowerCAmelCase	:       Dict		):
          @wraps(lowerCAmelCase		)
          def run_in_eager_mode(*lowerCAmelCase	:       Optional[Any]       ,    **lowerCAmelCase	:       Dict		):
               return func(*lowerCAmelCase       ,    **lowerCAmelCase		)

          @wraps(lowerCAmelCase		)
          @tf.function(experimental_compile=lowerCAmelCase		)
          def run_in_graph_mode(*lowerCAmelCase	:       Optional[Any]       ,    **lowerCAmelCase	:       Optional[Any]		):
               return func(*lowerCAmelCase       ,    **lowerCAmelCase		)

          if do_eager_mode is True:
               if use_xla is not False:
                    raise ValueError(
                        "Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`."		)
               return run_in_eager_mode
          else:
               return run_in_graph_mode

     return run_func



def 							a__ (				lowerCAmelCase	:       int       ,    lowerCAmelCase	:       int       ,    lowerCAmelCase	:       int		):






     '''simple docstring'''
     UpperCAmelCase__							: Dict 							=						random.Random()
     UpperCAmelCase__							: List[str] 							=						[rng.randint(0       ,    vocab_size - 1		) for i in range(batch_size * sequence_length		)]
     return tf.constant(lowerCAmelCase       ,    shape=(batch_size, sequence_length)       ,    dtype=tf.intaa		)







class   _lowercase						(       lowerCAmelCase_				):


     '''simple docstring'''





     _A								=							42
     _A								=							42
     _A								=							"TensorFlow"



     @property
     def  lowerCAmelCase__       (      self  )->					Optional[int]:
          return tf.__version__



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )->					float:
          # initialize GPU on separate process
          UpperCAmelCase__							: Any 							=						self.args.strategy
          if strategy is None:
               raise ValueError("A device strategy has to be initialized before using TensorFlow."  )
          UpperCAmelCase__							: Union[str, Any] 							=						self._prepare_inference_func(__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )
          return self._measure_speed(_inference  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )->					float:
          UpperCAmelCase__							: List[Any] 							=						self.args.strategy
          if strategy is None:
               raise ValueError("A device strategy has to be initialized before using TensorFlow."  )
          UpperCAmelCase__							: List[Any] 							=						self._prepare_train_func(__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )
          return self._measure_speed(_train  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )->					[Memory, Optional[MemorySummary]]:
          # initialize GPU on separate process
          if self.args.is_gpu:
               tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx] ,			__UpperCamelCase  )
          UpperCAmelCase__							: List[str] 							=						self.args.strategy
          if strategy is None:
               raise ValueError("A device strategy has to be initialized before using TensorFlow."  )
          UpperCAmelCase__							: Any 							=						self._prepare_inference_func(__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )
          return self._measure_memory(_inference  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )->					[Memory, Optional[MemorySummary]]:
          if self.args.is_gpu:
               tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx] ,			__UpperCamelCase  )
          UpperCAmelCase__							: Any 							=						self.args.strategy
          if strategy is None:
               raise ValueError("A device strategy has to be initialized before using TensorFlow."  )

          UpperCAmelCase__							: Optional[Any] 							=						self._prepare_train_func(__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )
          return self._measure_memory(_train  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )->					Callable[[], None]:
          UpperCAmelCase__							: Union[str, Any] 							=						self.config_dict[model_name]

          if self.args.fpaa:
               raise NotImplementedError("Mixed precision is currently not supported."  )

          UpperCAmelCase__							: Optional[int] 							=						(
              hasattr(__UpperCamelCase ,			"architectures"  )
              and isinstance(config.architectures ,			__UpperCamelCase  )
              and len(config.architectures  ) > 0
          )
          if not self.args.only_pretrain_model and has_model_class_in_config:
               try:
                    UpperCAmelCase__							: str 							=						"TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
                    UpperCAmelCase__							: Any 							=						__import__("transformers" ,			fromlist=[model_class]  )
                    UpperCAmelCase__							: List[Any] 							=						getattr(__UpperCamelCase ,			__UpperCamelCase  )
                    UpperCAmelCase__							: Dict 							=						model_cls(__UpperCamelCase  )
               except ImportError:
                    raise ImportError(
                        F"{model_class} does not exist. If you just want to test the pretrained model, you might want to"
                        " set `--only_pretrain_model` or `args.only_pretrain_model=True`."  )
          else:
               UpperCAmelCase__							: Any 							=						TF_MODEL_MAPPING[config.__class__](__UpperCamelCase  )

          # encoder-decoder has vocab size saved differently
          UpperCAmelCase__							: int 							=						config.vocab_size if hasattr(__UpperCamelCase ,			"vocab_size"  ) else config.encoder.vocab_size
          UpperCAmelCase__							: Optional[Any] 							=						random_input_ids(__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )

          @run_with_tf_optimizations(self.args.eager_mode ,			self.args.use_xla  )
          def encoder_decoder_forward():
               return model(__UpperCamelCase ,			decoder_input_ids=__UpperCamelCase ,			training=__UpperCamelCase  )

          @run_with_tf_optimizations(self.args.eager_mode ,			self.args.use_xla  )
          def encoder_forward():
               return model(__UpperCamelCase ,			training=__UpperCamelCase  )

          UpperCAmelCase__							: Dict 							=						encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

          return _inference



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )->					Callable[[], None]:
          UpperCAmelCase__							: List[Any] 							=						self.config_dict[model_name]

          if self.args.eager_mode is not False:
               raise ValueError("Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`."  )

          if self.args.fpaa:
               raise NotImplementedError("Mixed precision is currently not supported."  )

          UpperCAmelCase__							: Any 							=						(
              hasattr(__UpperCamelCase ,			"architectures"  )
              and isinstance(config.architectures ,			__UpperCamelCase  )
              and len(config.architectures  ) > 0
          )
          if not self.args.only_pretrain_model and has_model_class_in_config:
               try:
                    UpperCAmelCase__							: Any 							=						"TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
                    UpperCAmelCase__							: int 							=						__import__("transformers" ,			fromlist=[model_class]  )
                    UpperCAmelCase__							: int 							=						getattr(__UpperCamelCase ,			__UpperCamelCase  )
                    UpperCAmelCase__							: Optional[Any] 							=						model_cls(__UpperCamelCase  )
               except ImportError:
                    raise ImportError(
                        F"{model_class} does not exist. If you just want to test the pretrained model, you might want to"
                        " set `--only_pretrain_model` or `args.only_pretrain_model=True`."  )
          else:
               UpperCAmelCase__							: List[str] 							=						TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](__UpperCamelCase  )

          # encoder-decoder has vocab size saved differently
          UpperCAmelCase__							: Union[str, Any] 							=						config.vocab_size if hasattr(__UpperCamelCase ,			"vocab_size"  ) else config.encoder.vocab_size
          UpperCAmelCase__							: Dict 							=						random_input_ids(__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )

          @run_with_tf_optimizations(self.args.eager_mode ,			self.args.use_xla  )
          def encoder_decoder_train():
               UpperCAmelCase__							: Union[str, Any] 							=						model(__UpperCamelCase ,			decoder_input_ids=__UpperCamelCase ,			labels=__UpperCamelCase ,			training=__UpperCamelCase  )[0]
               UpperCAmelCase__							: Union[str, Any] 							=						tf.gradients(__UpperCamelCase ,			model.trainable_variables  )
               return gradients

          @run_with_tf_optimizations(self.args.eager_mode ,			self.args.use_xla  )
          def encoder_train():
               UpperCAmelCase__							: Union[str, Any] 							=						model(__UpperCamelCase ,			labels=__UpperCamelCase ,			training=__UpperCamelCase  )[0]
               UpperCAmelCase__							: Any 							=						tf.gradients(__UpperCamelCase ,			model.trainable_variables  )
               return gradients

          UpperCAmelCase__							: str 							=						encoder_decoder_train if config.is_encoder_decoder else encoder_train

          return _train



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					float:
          with self.args.strategy.scope():
               try:
                    if self.args.is_tpu or self.args.use_xla:
                         # run additional 10 times to stabilize compilation for tpu
                         logger.info("Do inference on TPU. Running model 5 times to stabilize compilation"  )
                         timeit.repeat(__UpperCamelCase ,			repeat=1 ,			number=5  )

                    # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
                    UpperCAmelCase__							: Optional[Any] 							=						timeit.repeat(
                        __UpperCamelCase ,			repeat=self.args.repeat ,			number=10 ,			)

                    return min(__UpperCamelCase  ) / 10.0
               except ResourceExhaustedError as e:
                    self.print_fn(F"Doesn't fit on GPU. {e}"  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					[Memory, MemorySummary]:
          logger.info(
              "Note that TensorFlow allocates more memory than "
              "it might need to speed up computation. "
              "The memory reported here corresponds to the memory "
              "reported by `nvidia-smi`, which can vary depending "
              "on total available memory on the GPU that is used."  )
          with self.args.strategy.scope():
               try:
                    if self.args.trace_memory_line_by_line:
                         if not self.args.eager_mode:
                              raise ValueError(
                                  "`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory"
                                  " consumption line by line."  )
                         UpperCAmelCase__							: List[str] 							=						start_memory_tracing("transformers"  )

                    if self.args.is_tpu:
                         # tpu
                         raise NotImplementedError(
                             "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking"
                             " with `args.memory=False`"  )
                    elif self.args.is_gpu:
                         # gpu
                         if not is_pyanvml_available():
                              logger.warning(
                                  "py3nvml not installed, we won't log GPU memory usage. "
                                  "Install py3nvml (pip install py3nvml) to log information about GPU."  )
                              UpperCAmelCase__							: Optional[int] 							=						"N/A"
                         else:
                              logger.info(
                                  "Measuring total GPU usage on GPU device. Make sure to not have additional processes"
                                  " running on the same GPU."  )
                              # init nvml
                              nvml.nvmlInit()
                              func()
                              UpperCAmelCase__							: Any 							=						nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx  )
                              UpperCAmelCase__							: Optional[int] 							=						nvml.nvmlDeviceGetMemoryInfo(__UpperCamelCase  )
                              UpperCAmelCase__							: str 							=						meminfo.used
                              UpperCAmelCase__							: int 							=						Memory(__UpperCamelCase  )
                              # shutdown nvml
                              nvml.nvmlShutdown()
                    else:
                         # cpu
                         if self.args.trace_memory_line_by_line:
                              logger.info(
                                  "When enabling line by line tracing, the max peak memory for CPU is inaccurate in"
                                  " TensorFlow."  )
                              UpperCAmelCase__							: Any 							=						None
                         else:
                              UpperCAmelCase__							: List[Any] 							=						measure_peak_memory_cpu(__UpperCamelCase  )
                              UpperCAmelCase__							: Optional[Any] 							=						Memory(__UpperCamelCase  ) if isinstance(__UpperCamelCase ,			__UpperCamelCase  ) else memory_bytes
                    if self.args.trace_memory_line_by_line:
                         UpperCAmelCase__							: Optional[Any] 							=						stop_memory_tracing(__UpperCamelCase  )
                         if memory is None:
                              UpperCAmelCase__							: Tuple 							=						summary.total
                    else:
                         UpperCAmelCase__							: int 							=						None

                    return memory, summary
               except ResourceExhaustedError as e:
                    self.print_fn(F"Doesn't fit on GPU. {e}"  )
                    return "N/A", None



