
"""simple docstring"""




import inspect
import unittest
from typing import List

import numpy as np

from transformers import EfficientFormerConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import (
						    TFEfficientFormerForImageClassification,
						    TFEfficientFormerForImageClassificationWithTeacher,
						    TFEfficientFormerModel,
						)
						from transformers.models.efficientformer.modeling_tf_efficientformer import (
						    TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
						)


if is_vision_available():
						from PIL import Image

						from transformers import EfficientFormerImageProcessor





class  snake_case     :

							def __init__(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_ = 13       ,       SCREAMING_SNAKE_CASE_ = 64       ,       SCREAMING_SNAKE_CASE_ = 2       ,       SCREAMING_SNAKE_CASE_ = 3       ,       SCREAMING_SNAKE_CASE_ = 3       ,       SCREAMING_SNAKE_CASE_ = True       ,       SCREAMING_SNAKE_CASE_ = True       ,       SCREAMING_SNAKE_CASE_ = 1_28       ,       SCREAMING_SNAKE_CASE_=[16, 32, 64, 1_28]       ,       SCREAMING_SNAKE_CASE_ = 7       ,       SCREAMING_SNAKE_CASE_ = 4       ,       SCREAMING_SNAKE_CASE_ = 37       ,       SCREAMING_SNAKE_CASE_ = "gelu"       ,       SCREAMING_SNAKE_CASE_ = 0.1       ,       SCREAMING_SNAKE_CASE_ = 0.1       ,       SCREAMING_SNAKE_CASE_ = 10       ,       SCREAMING_SNAKE_CASE_ = 0.02       ,       SCREAMING_SNAKE_CASE_ = 2       ,       SCREAMING_SNAKE_CASE_ = 1       ,       SCREAMING_SNAKE_CASE_ = 1_28       ,       SCREAMING_SNAKE_CASE_ = [2, 2, 2, 2]       ,       SCREAMING_SNAKE_CASE_ = 2       ,       SCREAMING_SNAKE_CASE_ = 2       ,       ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = parent
									SCREAMING_SNAKE_CASE_						 = batch_size
									SCREAMING_SNAKE_CASE_						 = image_size
									SCREAMING_SNAKE_CASE_						 = patch_size
									SCREAMING_SNAKE_CASE_						 = num_channels
									SCREAMING_SNAKE_CASE_						 = is_training
									SCREAMING_SNAKE_CASE_						 = use_labels
									SCREAMING_SNAKE_CASE_						 = hidden_size
									SCREAMING_SNAKE_CASE_						 = num_hidden_layers
									SCREAMING_SNAKE_CASE_						 = num_attention_heads
									SCREAMING_SNAKE_CASE_						 = intermediate_size
									SCREAMING_SNAKE_CASE_						 = hidden_act
									SCREAMING_SNAKE_CASE_						 = hidden_dropout_prob
									SCREAMING_SNAKE_CASE_						 = attention_probs_dropout_prob
									SCREAMING_SNAKE_CASE_						 = type_sequence_label_size
									SCREAMING_SNAKE_CASE_						 = initializer_range
									SCREAMING_SNAKE_CASE_						 = encoder_stride
									SCREAMING_SNAKE_CASE_						 = num_attention_outputs
									SCREAMING_SNAKE_CASE_						 = embed_dim
									SCREAMING_SNAKE_CASE_						 = embed_dim + 1
									SCREAMING_SNAKE_CASE_						 = resolution
									SCREAMING_SNAKE_CASE_						 = depths
									SCREAMING_SNAKE_CASE_						 = hidden_sizes
									SCREAMING_SNAKE_CASE_						 = dim
									SCREAMING_SNAKE_CASE_						 = mlp_expansion_ratio

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]    )

									SCREAMING_SNAKE_CASE_						 = None
									if self.use_labels:
											SCREAMING_SNAKE_CASE_						 = ids_tensor([self.batch_size]       ,       self.type_sequence_label_size    )

									SCREAMING_SNAKE_CASE_						 = self.get_config()

									return config, pixel_values, labels

							def _lowercase					(self    ):

									"""simple docstring"""







									return EfficientFormerConfig(
									    image_size=self.image_size       ,       patch_size=self.patch_size       ,       num_channels=self.num_channels       ,       hidden_size=self.hidden_size       ,       num_hidden_layers=self.num_hidden_layers       ,       num_attention_heads=self.num_attention_heads       ,       intermediate_size=self.intermediate_size       ,       hidden_act=self.hidden_act       ,       hidden_dropout_prob=self.hidden_dropout_prob       ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,       is_decoder=SCREAMING_SNAKE_CASE_       ,       initializer_range=self.initializer_range       ,       encoder_stride=self.encoder_stride       ,       resolution=self.resolution       ,       depths=self.depths       ,       hidden_sizes=self.hidden_sizes       ,       dim=self.dim       ,       mlp_expansion_ratio=self.mlp_expansion_ratio       ,       )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TFEfficientFormerModel(config=SCREAMING_SNAKE_CASE_    )
									SCREAMING_SNAKE_CASE_						 = model(SCREAMING_SNAKE_CASE_       ,       training=SCREAMING_SNAKE_CASE_    )
									self.parent.assertEqual(result.last_hidden_state.shape       ,       (self.batch_size, self.seq_length, self.hidden_size)    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = self.type_sequence_label_size
									SCREAMING_SNAKE_CASE_						 = TFEfficientFormerForImageClassification(SCREAMING_SNAKE_CASE_    )
									SCREAMING_SNAKE_CASE_						 = model(SCREAMING_SNAKE_CASE_       ,       labels=SCREAMING_SNAKE_CASE_       ,       training=SCREAMING_SNAKE_CASE_    )
									self.parent.assertEqual(result.logits.shape       ,       (self.batch_size, self.type_sequence_label_size)    )

									# test greyscale images
									SCREAMING_SNAKE_CASE_						 = 1
									SCREAMING_SNAKE_CASE_						 = TFEfficientFormerForImageClassification(SCREAMING_SNAKE_CASE_    )

									SCREAMING_SNAKE_CASE_						 = floats_tensor([self.batch_size, 1, self.image_size, self.image_size]    )
									SCREAMING_SNAKE_CASE_						 = model(SCREAMING_SNAKE_CASE_       ,       labels=SCREAMING_SNAKE_CASE_    )
									self.parent.assertEqual(result.logits.shape       ,       (self.batch_size, self.type_sequence_label_size)    )







							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = self.prepare_config_and_inputs()
									SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_						 = config_and_inputs
									SCREAMING_SNAKE_CASE_						 = {'''pixel_values''': pixel_values}
									return config, inputs_dict







@require_tf
class  snake_case     (							__lowercase       ,			__lowercase       ,			unittest.TestCase   ):
							UpperCAmelCase__						     =      (
							    (
							        TFEfficientFormerModel,
							        TFEfficientFormerForImageClassificationWithTeacher,
							        TFEfficientFormerForImageClassification,
							    )
							    if is_tf_available()
							    else ()
							)
							UpperCAmelCase__						     =      (
							    {
							        '''feature-extraction''': TFEfficientFormerModel,
							        '''image-classification''': (
							            TFEfficientFormerForImageClassification,
							            TFEfficientFormerForImageClassificationWithTeacher,
							        ),
							    }
							    if is_tf_available()
							    else {}
							)

							UpperCAmelCase__						     =      False

							UpperCAmelCase__						     =      False
							UpperCAmelCase__						     =      False
							UpperCAmelCase__						     =      False
							UpperCAmelCase__						     =      False

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TFEfficientFormerModelTester(self    )
									SCREAMING_SNAKE_CASE_						 = ConfigTester(
									    self       ,       config_class=SCREAMING_SNAKE_CASE_       ,       has_text_modality=SCREAMING_SNAKE_CASE_       ,       hidden_size=37    )

							def _lowercase					(self    ):

									"""simple docstring"""







									self.config_tester.run_common_tests()

							@unittest.skip(reason='''EfficientFormer does not use inputs_embeds'''    )
							def _lowercase					(self    ):

									"""simple docstring"""







									pass

							@unittest.skip(reason='''EfficientFormer does not support input and output embeddings'''    )
							def _lowercase					(self    ):

									"""simple docstring"""







									pass

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_						 = self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											SCREAMING_SNAKE_CASE_						 = model_class(SCREAMING_SNAKE_CASE_    )
											SCREAMING_SNAKE_CASE_						 = inspect.signature(model.call    )
											# signature.parameters is an OrderedDict => so arg_names order is deterministic
											SCREAMING_SNAKE_CASE_						 = [*signature.parameters.keys()]

											SCREAMING_SNAKE_CASE_						 = ['''pixel_values''']
											self.assertListEqual(arg_names[:1]       ,       SCREAMING_SNAKE_CASE_    )

							def _lowercase					(self    ):

									"""simple docstring"""







									def check_hidden_states_output(SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ):
											SCREAMING_SNAKE_CASE_						 = model_class(SCREAMING_SNAKE_CASE_    )

											SCREAMING_SNAKE_CASE_						 = model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    )       ,       training=SCREAMING_SNAKE_CASE_    )
											SCREAMING_SNAKE_CASE_						 = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

											SCREAMING_SNAKE_CASE_						 = getattr(
											    self.model_tester       ,       '''expected_num_hidden_layers'''       ,       self.model_tester.num_hidden_layers + 1    )
											self.assertEqual(len(SCREAMING_SNAKE_CASE_    )       ,       SCREAMING_SNAKE_CASE_    )

											if hasattr(self.model_tester       ,       '''encoder_seq_length'''    ):
													SCREAMING_SNAKE_CASE_						 = self.model_tester.encoder_seq_length
													if hasattr(self.model_tester       ,       '''chunk_length'''    ) and self.model_tester.chunk_length > 1:
															SCREAMING_SNAKE_CASE_						 = seq_length * self.model_tester.chunk_length
											else:
													SCREAMING_SNAKE_CASE_						 = self.model_tester.seq_length

											self.assertListEqual(
											    list(hidden_states[-1].shape[-2:]    )       ,       [seq_length, self.model_tester.hidden_size]       ,       )

											if config.is_encoder_decoder:
													SCREAMING_SNAKE_CASE_						 = outputs.decoder_hidden_states

													self.asseretIsInstance(SCREAMING_SNAKE_CASE_       ,       (list, tuple)    )
													self.assertEqual(len(SCREAMING_SNAKE_CASE_    )       ,       SCREAMING_SNAKE_CASE_    )
													SCREAMING_SNAKE_CASE_						 = getattr(self.model_tester       ,       '''seq_length'''       ,       SCREAMING_SNAKE_CASE_    )
													SCREAMING_SNAKE_CASE_						 = getattr(self.model_tester       ,       '''decoder_seq_length'''       ,       SCREAMING_SNAKE_CASE_    )

													self.assertListEqual(
													    list(hidden_states[-1].shape[-2:]    )       ,       [decoder_seq_length, self.model_tester.hidden_size]       ,       )

									SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_						 = self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											SCREAMING_SNAKE_CASE_						 = True
											check_hidden_states_output(SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    )

											# check that output_hidden_states also work using config
											del inputs_dict["output_hidden_states"]
											SCREAMING_SNAKE_CASE_						 = True

											check_hidden_states_output(SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_=False    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = super()._prepare_for_class(SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       return_labels=SCREAMING_SNAKE_CASE_    )

									if return_labels:
											if model_class.__name__ == "TFEfficientFormerForImageClassificationWithTeacher":
													del inputs_dict["labels"]

									return inputs_dict

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_    )

							@unittest.skip(reason='''EfficientFormer does not implement masked image modeling yet'''    )
							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_image_modeling(*SCREAMING_SNAKE_CASE_    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_image_classification(*SCREAMING_SNAKE_CASE_    )

							@slow
							def _lowercase					(self    ):

									"""simple docstring"""







									for model_name in TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											SCREAMING_SNAKE_CASE_						 = TFEfficientFormerModel.from_pretrained(SCREAMING_SNAKE_CASE_    )
											self.assertIsNotNone(SCREAMING_SNAKE_CASE_    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_						 = self.model_tester.prepare_config_and_inputs_for_common()
									SCREAMING_SNAKE_CASE_						 = True

									SCREAMING_SNAKE_CASE_						 = getattr(self.model_tester       ,       '''seq_length'''       ,       SCREAMING_SNAKE_CASE_    )
									SCREAMING_SNAKE_CASE_						 = getattr(self.model_tester       ,       '''encoder_seq_length'''       ,       SCREAMING_SNAKE_CASE_    )
									SCREAMING_SNAKE_CASE_						 = getattr(self.model_tester       ,       '''key_length'''       ,       SCREAMING_SNAKE_CASE_    )
									SCREAMING_SNAKE_CASE_						 = getattr(self.model_tester       ,       '''chunk_length'''       ,       SCREAMING_SNAKE_CASE_    )

									if chunk_length is not None and hasattr(self.model_tester       ,       '''num_hashes'''    ):
											SCREAMING_SNAKE_CASE_						 = encoder_seq_length * self.model_tester.num_hashes

									for model_class in self.all_model_classes:
											SCREAMING_SNAKE_CASE_						 = True
											SCREAMING_SNAKE_CASE_						 = False
											SCREAMING_SNAKE_CASE_						 = True
											SCREAMING_SNAKE_CASE_						 = model_class(SCREAMING_SNAKE_CASE_    )

											SCREAMING_SNAKE_CASE_						 = model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    )       ,       training=SCREAMING_SNAKE_CASE_    )
											SCREAMING_SNAKE_CASE_						 = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
											self.assertEqual(len(SCREAMING_SNAKE_CASE_    )       ,       self.model_tester.num_attention_outputs    )

											# check that output_attentions also work using config
											del inputs_dict["output_attentions"]
											SCREAMING_SNAKE_CASE_						 = True
											SCREAMING_SNAKE_CASE_						 = model_class(SCREAMING_SNAKE_CASE_    )
											SCREAMING_SNAKE_CASE_						 = model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    )       ,       training=SCREAMING_SNAKE_CASE_    )

											SCREAMING_SNAKE_CASE_						 = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
											self.assertEqual(len(SCREAMING_SNAKE_CASE_    )       ,       self.model_tester.num_attention_outputs    )

											if chunk_length is not None:
													self.assertListEqual(
													    list(attentions[0].shape[-4:]    )       ,       [self.model_tester.num_attention_heads, encoder_seq_length, chunk_length, encoder_key_length]       ,       )
											else:
													self.assertListEqual(
													    list(attentions[0].shape[-3:]    )       ,       [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length]       ,       )







							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_						 = self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											# Prepare our model
											SCREAMING_SNAKE_CASE_						 = model_class(SCREAMING_SNAKE_CASE_    )
											# These are maximally general inputs for the model, with multiple None dimensions
											# Hopefully this will catch any conditionals that fail for flexible shapes
											SCREAMING_SNAKE_CASE_						 = {
											    key: tf.keras.Input(shape=val.shape[1:]       ,       dtype=val.dtype       ,       name=SCREAMING_SNAKE_CASE_    )
											    for key, val in model.input_signature.items()
											    if key in model.dummy_inputs
											}
											SCREAMING_SNAKE_CASE_						 = model(SCREAMING_SNAKE_CASE_    )
											self.assertTrue(outputs_dict is not None    )







def        _lowerCamelCase ( ):
		SCREAMING_SNAKE_CASE_						 = Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''    )
		return image





@require_tf
@require_vision
class  snake_case     (							unittest.TestCase   ):

							@cached_property
							def _lowercase					(self    ):

									"""simple docstring"""







									return (
									    EfficientFormerImageProcessor.from_pretrained('''snap-research/efficientformer-l1-300'''    )
									    if is_vision_available()
									    else None
									)

							@slow
							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TFEfficientFormerForImageClassification.from_pretrained('''snap-research/efficientformer-l1-300'''    )
									SCREAMING_SNAKE_CASE_						 = self.default_image_processor
									SCREAMING_SNAKE_CASE_						 = prepare_img()
									SCREAMING_SNAKE_CASE_						 = image_processor(images=SCREAMING_SNAKE_CASE_       ,       return_tensors='''tf'''    )
									# forward pass
									SCREAMING_SNAKE_CASE_						 = model(**SCREAMING_SNAKE_CASE_       ,       training=SCREAMING_SNAKE_CASE_    )
									# verify the logits
									SCREAMING_SNAKE_CASE_						 = tf.TensorShape((1, 10_00)    )
									self.assertEqual(outputs.logits.shape       ,       SCREAMING_SNAKE_CASE_    )
									SCREAMING_SNAKE_CASE_						 = tf.constant([-0.05_55, 0.48_25, -0.08_52]    )
									self.assertTrue(np.allclose(outputs.logits[0, :3]       ,       SCREAMING_SNAKE_CASE_       ,       atol=1e-4    )    )







							@slow
							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TFEfficientFormerForImageClassificationWithTeacher.from_pretrained(
									    '''snap-research/efficientformer-l1-300'''    )
									SCREAMING_SNAKE_CASE_						 = self.default_image_processor
									SCREAMING_SNAKE_CASE_						 = prepare_img()
									SCREAMING_SNAKE_CASE_						 = image_processor(images=SCREAMING_SNAKE_CASE_       ,       return_tensors='''tf'''    )
									# forward pass
									SCREAMING_SNAKE_CASE_						 = model(**SCREAMING_SNAKE_CASE_       ,       training=SCREAMING_SNAKE_CASE_    )
									# verify the logits
									SCREAMING_SNAKE_CASE_						 = tf.TensorShape((1, 10_00)    )
									self.assertEqual(outputs.logits.shape       ,       SCREAMING_SNAKE_CASE_    )
									SCREAMING_SNAKE_CASE_						 = tf.constant([-0.13_12, 0.43_53, -1.04_99]    )
									self.assertTrue(np.allclose(outputs.logits[0, :3]       ,       SCREAMING_SNAKE_CASE_       ,       atol=1e-4    )    )

"""simple docstring"""




import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin





class  snake_case     (							__lowercase       ,			unittest.TestCase   ):
							UpperCAmelCase__						     =      TransfoXLTokenizer
							UpperCAmelCase__						     =      False
							UpperCAmelCase__						     =      False

							def _lowercase					(self    ):

									"""simple docstring"""







									super().setUp()

									SCREAMING_SNAKE_CASE_						 = [
									    '''<unk>''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''want''',
									    '''unwanted''',
									    '''wa''',
									    '''un''',
									    '''running''',
									    ''',''',
									    '''low''',
									    '''l''',
									]
									SCREAMING_SNAKE_CASE_						 = os.path.join(self.tmpdirname       ,       VOCAB_FILES_NAMES['''vocab_file''']    )
									with open(self.vocab_file       ,       '''w'''       ,       encoding='''utf-8'''    ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )

							def _lowercase					(self       ,       **SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = True
									return TransfoXLTokenizer.from_pretrained(self.tmpdirname       ,       **SCREAMING_SNAKE_CASE_    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = '''<unk> UNwanted , running'''
									SCREAMING_SNAKE_CASE_						 = '''<unk> unwanted, running'''
									return input_text, output_text

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TransfoXLTokenizer(vocab_file=self.vocab_file       ,       lower_case=SCREAMING_SNAKE_CASE_    )

									SCREAMING_SNAKE_CASE_						 = tokenizer.tokenize('''<unk> UNwanted , running'''    )
									self.assertListEqual(SCREAMING_SNAKE_CASE_       ,       ['''<unk>''', '''unwanted''', ''',''', '''running''']    )

									self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_    )       ,       [0, 4, 8, 7]    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TransfoXLTokenizer(lower_case=SCREAMING_SNAKE_CASE_    )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  '''    )       ,       ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TransfoXLTokenizer(lower_case=SCREAMING_SNAKE_CASE_    )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  '''    )       ,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TransfoXLTokenizer(lower_case=SCREAMING_SNAKE_CASE_    )
									SCREAMING_SNAKE_CASE_						 = '''Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'''
									SCREAMING_SNAKE_CASE_						 = [
									    '''Hello''',
									    '''(''',
									    '''bracket''',
									    ''')''',
									    '''and''',
									    '''side''',
									    '''@-@''',
									    '''scrolled''',
									    '''[''',
									    '''and''',
									    ''']''',
									    '''Henry''',
									    '''\'s''',
									    '''$''',
									    '''5''',
									    '''@,@''',
									    '''000''',
									    '''with''',
									    '''3''',
									    '''@.@''',
									    '''34''',
									    '''m''',
									    '''.''',
									    '''What''',
									    '''\'s''',
									    '''up''',
									    '''!''',
									    '''?''',
									]

									self.assertListEqual(tokenizer.tokenize(SCREAMING_SNAKE_CASE_    )       ,       SCREAMING_SNAKE_CASE_    )

									self.assertEqual(tokenizer.convert_tokens_to_string(SCREAMING_SNAKE_CASE_    )       ,       SCREAMING_SNAKE_CASE_    )







							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = self.get_tokenizer()
									SCREAMING_SNAKE_CASE_						 = len(SCREAMING_SNAKE_CASE_    )

									tokenizer.add_tokens(['''new1''', '''new2''']    )
									tokenizer.move_added_token('''new1'''       ,       1    )

									# Check that moved token is not copied (duplicate)
									self.assertEqual(len(SCREAMING_SNAKE_CASE_    )       ,       original_len + 2    )
									# Check that token is moved to specified id
									self.assertEqual(tokenizer.encode('''new1'''    )       ,       [1]    )
									self.assertEqual(tokenizer.decode([1]    )       ,       '''new1'''    )