
"""simple docstring"""




import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
						import tensorflow as tf
						from tensorflow.python.framework.errors_impl import ResourceExhaustedError

						from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
						import pyanvml.pyanvml as nvml

lowerCAmelCase__							=				logging.get_logger(__name__)



def        _lowerCamelCase ( __a,							__a    ):
		def run_func(__a    ):
				@wraps(__A    )
				def run_in_eager_mode(*__a,							**__a    ):
						return func(*__A,							**__A    )

				@wraps(__A    )
				@tf.function(experimental_compile=__A    )
				def run_in_graph_mode(*__a,							**__a    ):
						return func(*__A,							**__A    )

				if do_eager_mode is True:
						if use_xla is not False:
								raise ValueError(
								    '''Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.'''    )
						return run_in_eager_mode
				else:
						return run_in_graph_mode

		return run_func



def        _lowerCamelCase ( __a,							__a,							__a    ):
		SCREAMING_SNAKE_CASE_						 = random.Random()
		SCREAMING_SNAKE_CASE_						 = [rng.randint(0,							vocab_size - 1    ) for i in range(batch_size * sequence_length    )]
		return tf.constant(__A,							shape=(batch_size, sequence_length),							dtype=tf.intaa    )





class  snake_case     (							__lowerCAmelCase   ):
							UpperCAmelCase__						     =      42
							UpperCAmelCase__						     =      42
							UpperCAmelCase__						     =      '''TensorFlow'''

							@property
							def _lowercase					(self    ):

									"""simple docstring"""







									return tf.__version__

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = self.args.strategy
									if strategy is None:
											raise ValueError('''A device strategy has to be initialized before using TensorFlow.'''    )
									SCREAMING_SNAKE_CASE_						 = self._prepare_inference_func(_UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase    )
									return self._measure_speed(_inference    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = self.args.strategy
									if strategy is None:
											raise ValueError('''A device strategy has to be initialized before using TensorFlow.'''    )
									SCREAMING_SNAKE_CASE_						 = self._prepare_train_func(_UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase    )
									return self._measure_speed(_train    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									if self.args.is_gpu:
											tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]       ,       _UpperCamelCase    )
									SCREAMING_SNAKE_CASE_						 = self.args.strategy
									if strategy is None:
											raise ValueError('''A device strategy has to be initialized before using TensorFlow.'''    )
									SCREAMING_SNAKE_CASE_						 = self._prepare_inference_func(_UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase    )
									return self._measure_memory(_inference    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									if self.args.is_gpu:
											tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]       ,       _UpperCamelCase    )
									SCREAMING_SNAKE_CASE_						 = self.args.strategy
									if strategy is None:
											raise ValueError('''A device strategy has to be initialized before using TensorFlow.'''    )

									SCREAMING_SNAKE_CASE_						 = self._prepare_train_func(_UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase    )
									return self._measure_memory(_train    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = self.config_dict[model_name]

									if self.args.fpaa:
											raise NotImplementedError('''Mixed precision is currently not supported.'''    )

									SCREAMING_SNAKE_CASE_						 = (
									    hasattr(_UpperCamelCase       ,       '''architectures'''    )
									    and isinstance(config.architectures       ,       _UpperCamelCase    )
									    and len(config.architectures    ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
											try:
													SCREAMING_SNAKE_CASE_						 = """TF""" + config.architectures[0]  # prepend 'TF' for tensorflow model
													SCREAMING_SNAKE_CASE_						 = __import__('''transformers'''       ,       fromlist=[model_class]    )
													SCREAMING_SNAKE_CASE_						 = getattr(_UpperCamelCase       ,       _UpperCamelCase    )
													SCREAMING_SNAKE_CASE_						 = model_cls(_UpperCamelCase    )
											except ImportError:
													raise ImportError(
													    f'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
													    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''    )
									else:
											SCREAMING_SNAKE_CASE_						 = TF_MODEL_MAPPING[config.__class__](_UpperCamelCase    )

									# encoder-decoder has vocab size saved differently
									SCREAMING_SNAKE_CASE_						 = config.vocab_size if hasattr(_UpperCamelCase       ,       '''vocab_size'''    ) else config.encoder.vocab_size
									SCREAMING_SNAKE_CASE_						 = random_input_ids(_UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase    )

									@run_with_tf_optimizations(self.args.eager_mode       ,       self.args.use_xla    )
									def encoder_decoder_forward():
											return model(_UpperCamelCase       ,       decoder_input_ids=_UpperCamelCase       ,       training=_UpperCamelCase    )

									@run_with_tf_optimizations(self.args.eager_mode       ,       self.args.use_xla    )
									def encoder_forward():
											return model(_UpperCamelCase       ,       training=_UpperCamelCase    )

									SCREAMING_SNAKE_CASE_						 = encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

									return _inference

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = self.config_dict[model_name]

									if self.args.eager_mode is not False:
											raise ValueError('''Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.'''    )

									if self.args.fpaa:
											raise NotImplementedError('''Mixed precision is currently not supported.'''    )

									SCREAMING_SNAKE_CASE_						 = (
									    hasattr(_UpperCamelCase       ,       '''architectures'''    )
									    and isinstance(config.architectures       ,       _UpperCamelCase    )
									    and len(config.architectures    ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
											try:
													SCREAMING_SNAKE_CASE_						 = """TF""" + config.architectures[0]  # prepend 'TF' for tensorflow model
													SCREAMING_SNAKE_CASE_						 = __import__('''transformers'''       ,       fromlist=[model_class]    )
													SCREAMING_SNAKE_CASE_						 = getattr(_UpperCamelCase       ,       _UpperCamelCase    )
													SCREAMING_SNAKE_CASE_						 = model_cls(_UpperCamelCase    )
											except ImportError:
													raise ImportError(
													    f'{model_class} does not exist. If you just want to test the pretrained model, you might want to'
													    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''    )
									else:
											SCREAMING_SNAKE_CASE_						 = TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_UpperCamelCase    )

									# encoder-decoder has vocab size saved differently
									SCREAMING_SNAKE_CASE_						 = config.vocab_size if hasattr(_UpperCamelCase       ,       '''vocab_size'''    ) else config.encoder.vocab_size
									SCREAMING_SNAKE_CASE_						 = random_input_ids(_UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase    )

									@run_with_tf_optimizations(self.args.eager_mode       ,       self.args.use_xla    )
									def encoder_decoder_train():
											SCREAMING_SNAKE_CASE_						 = model(_UpperCamelCase       ,       decoder_input_ids=_UpperCamelCase       ,       labels=_UpperCamelCase       ,       training=_UpperCamelCase    )[0]
											SCREAMING_SNAKE_CASE_						 = tf.gradients(_UpperCamelCase       ,       model.trainable_variables    )
											return gradients

									@run_with_tf_optimizations(self.args.eager_mode       ,       self.args.use_xla    )
									def encoder_train():
											SCREAMING_SNAKE_CASE_						 = model(_UpperCamelCase       ,       labels=_UpperCamelCase       ,       training=_UpperCamelCase    )[0]
											SCREAMING_SNAKE_CASE_						 = tf.gradients(_UpperCamelCase       ,       model.trainable_variables    )
											return gradients

									SCREAMING_SNAKE_CASE_						 = encoder_decoder_train if config.is_encoder_decoder else encoder_train

									return _train

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									with self.args.strategy.scope():
											try:
													if self.args.is_tpu or self.args.use_xla:
															# run additional 10 times to stabilize compilation for tpu
															logger.info('''Do inference on TPU. Running model 5 times to stabilize compilation'''    )
															timeit.repeat(_UpperCamelCase       ,       repeat=1       ,       number=5    )

													# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
													SCREAMING_SNAKE_CASE_						 = timeit.repeat(
													    _UpperCamelCase       ,       repeat=self.args.repeat       ,       number=10       ,       )

													return min(_UpperCamelCase    ) / 10.0
											except ResourceExhaustedError as e:
													self.print_fn(f'Doesn\'t fit on GPU. {e}'    )







							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									logger.info(
									    '''Note that TensorFlow allocates more memory than '''
									    '''it might need to speed up computation. '''
									    '''The memory reported here corresponds to the memory '''
									    '''reported by `nvidia-smi`, which can vary depending '''
									    '''on total available memory on the GPU that is used.'''    )
									with self.args.strategy.scope():
											try:
													if self.args.trace_memory_line_by_line:
															if not self.args.eager_mode:
																	raise ValueError(
																	    '''`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory'''
																	    ''' consumption line by line.'''    )
															SCREAMING_SNAKE_CASE_						 = start_memory_tracing('''transformers'''    )

													if self.args.is_tpu:
															# tpu
															raise NotImplementedError(
															    '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking'''
															    ''' with `args.memory=False`'''    )
													elif self.args.is_gpu:
															# gpu
															if not is_pyanvml_available():
																	logger.warning(
																	    '''py3nvml not installed, we won\'t log GPU memory usage. '''
																	    '''Install py3nvml (pip install py3nvml) to log information about GPU.'''    )
																	SCREAMING_SNAKE_CASE_						 = """N/A"""
															else:
																	logger.info(
																	    '''Measuring total GPU usage on GPU device. Make sure to not have additional processes'''
																	    ''' running on the same GPU.'''    )
																	# init nvml
																	nvml.nvmlInit()
																	func()
																	SCREAMING_SNAKE_CASE_						 = nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx    )
																	SCREAMING_SNAKE_CASE_						 = nvml.nvmlDeviceGetMemoryInfo(_UpperCamelCase    )
																	SCREAMING_SNAKE_CASE_						 = meminfo.used
																	SCREAMING_SNAKE_CASE_						 = Memory(_UpperCamelCase    )
																	# shutdown nvml
																	nvml.nvmlShutdown()
													else:
															# cpu
															if self.args.trace_memory_line_by_line:
																	logger.info(
																	    '''When enabling line by line tracing, the max peak memory for CPU is inaccurate in'''
																	    ''' TensorFlow.'''    )
																	SCREAMING_SNAKE_CASE_						 = None
															else:
																	SCREAMING_SNAKE_CASE_						 = measure_peak_memory_cpu(_UpperCamelCase    )
																	SCREAMING_SNAKE_CASE_						 = Memory(_UpperCamelCase    ) if isinstance(_UpperCamelCase       ,       _UpperCamelCase    ) else memory_bytes
													if self.args.trace_memory_line_by_line:
															SCREAMING_SNAKE_CASE_						 = stop_memory_tracing(_UpperCamelCase    )
															if memory is None:
																	SCREAMING_SNAKE_CASE_						 = summary.total
													else:
															SCREAMING_SNAKE_CASE_						 = None

													return memory, summary
											except ResourceExhaustedError as e:
													self.print_fn(f'Doesn\'t fit on GPU. {e}'    )
													return "N/A", None


"""simple docstring"""




import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin





class  snake_case     (							__lowercase       ,			unittest.TestCase   ):
							UpperCAmelCase__						     =      TransfoXLTokenizer
							UpperCAmelCase__						     =      False
							UpperCAmelCase__						     =      False

							def _lowercase					(self    ):

									"""simple docstring"""







									super().setUp()

									SCREAMING_SNAKE_CASE_						 = [
									    '''<unk>''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''want''',
									    '''unwanted''',
									    '''wa''',
									    '''un''',
									    '''running''',
									    ''',''',
									    '''low''',
									    '''l''',
									]
									SCREAMING_SNAKE_CASE_						 = os.path.join(self.tmpdirname       ,       VOCAB_FILES_NAMES['''vocab_file''']    )
									with open(self.vocab_file       ,       '''w'''       ,       encoding='''utf-8'''    ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )

							def _lowercase					(self       ,       **SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = True
									return TransfoXLTokenizer.from_pretrained(self.tmpdirname       ,       **SCREAMING_SNAKE_CASE_    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = '''<unk> UNwanted , running'''
									SCREAMING_SNAKE_CASE_						 = '''<unk> unwanted, running'''
									return input_text, output_text

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TransfoXLTokenizer(vocab_file=self.vocab_file       ,       lower_case=SCREAMING_SNAKE_CASE_    )

									SCREAMING_SNAKE_CASE_						 = tokenizer.tokenize('''<unk> UNwanted , running'''    )
									self.assertListEqual(SCREAMING_SNAKE_CASE_       ,       ['''<unk>''', '''unwanted''', ''',''', '''running''']    )

									self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_    )       ,       [0, 4, 8, 7]    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TransfoXLTokenizer(lower_case=SCREAMING_SNAKE_CASE_    )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  '''    )       ,       ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TransfoXLTokenizer(lower_case=SCREAMING_SNAKE_CASE_    )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  '''    )       ,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TransfoXLTokenizer(lower_case=SCREAMING_SNAKE_CASE_    )
									SCREAMING_SNAKE_CASE_						 = '''Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'''
									SCREAMING_SNAKE_CASE_						 = [
									    '''Hello''',
									    '''(''',
									    '''bracket''',
									    ''')''',
									    '''and''',
									    '''side''',
									    '''@-@''',
									    '''scrolled''',
									    '''[''',
									    '''and''',
									    ''']''',
									    '''Henry''',
									    '''\'s''',
									    '''$''',
									    '''5''',
									    '''@,@''',
									    '''000''',
									    '''with''',
									    '''3''',
									    '''@.@''',
									    '''34''',
									    '''m''',
									    '''.''',
									    '''What''',
									    '''\'s''',
									    '''up''',
									    '''!''',
									    '''?''',
									]

									self.assertListEqual(tokenizer.tokenize(SCREAMING_SNAKE_CASE_    )       ,       SCREAMING_SNAKE_CASE_    )

									self.assertEqual(tokenizer.convert_tokens_to_string(SCREAMING_SNAKE_CASE_    )       ,       SCREAMING_SNAKE_CASE_    )







							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = self.get_tokenizer()
									SCREAMING_SNAKE_CASE_						 = len(SCREAMING_SNAKE_CASE_    )

									tokenizer.add_tokens(['''new1''', '''new2''']    )
									tokenizer.move_added_token('''new1'''       ,       1    )

									# Check that moved token is not copied (duplicate)
									self.assertEqual(len(SCREAMING_SNAKE_CASE_    )       ,       original_len + 2    )
									# Check that token is moved to specified id
									self.assertEqual(tokenizer.encode('''new1'''    )       ,       [1]    )
									self.assertEqual(tokenizer.decode([1]    )       ,       '''new1'''    )