



import random
import unittest

import numpy as np

import transformers
from transformers import is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax


if is_flax_available():
   import os

   import jax.numpy as jnp
   from jax import jit

   from transformers import AutoTokenizer, FlaxAutoModelForCausalLM
   from transformers.modeling_flax_pytorch_utils import load_flax_weights_in_pytorch_model

   UpperCamelCase__ =						"0.12"  # assumed parallelism: 8


if is_torch_available():
   import torch






def  UpperCAmelCase__    (    lowercase__						,       lowercase__						,       lowercase__=None   ) ->      List[Any]:
      if rng is None:
            __lowercase					=				random.Random()

      __lowercase					=				1
      for dim in shape:
            total_dims *= dim

      __lowercase					=				[]
      for _ in range(_lowerCAmelCase   ):
            values.append(rng.randint(0						,       vocab_size - 1   )   )

      __lowercase					=				np.array(_lowerCAmelCase						,       dtype=jnp.intaa   ).reshape(_lowerCAmelCase   )

      return output






def  UpperCAmelCase__    (    lowercase__						,       lowercase__=None   ) ->      Optional[Any]:
      __lowercase					=				ids_tensor(_lowerCAmelCase						,       vocab_size=2						,       rng=_lowerCAmelCase   )
      # make sure that at least one token is attended to for each batch
      __lowercase					=				1
      return attn_mask
@require_flax
class 			_lowerCAmelCase   :






   """simple docstring"""





   lowercase__      :  Optional[int]      =			None
   lowercase__      :  List[Any]      =			()





   def 		snake_case__			(				self	:       Any   )						->					Dict:
         """simple docstring"""







         __lowercase	,			__lowercase					=				self.model_tester.prepare_config_and_inputs_for_common()

         # cut to half length & take max batch_size 3
         __lowercase					=				2
         __lowercase					=				inputs["""input_ids"""].shape[-1] // 2
         __lowercase					=				inputs["""input_ids"""][:max_batch_size, :sequence_length]

         __lowercase					=				jnp.ones_like(_lowerCamelCase   )
         __lowercase					=				attention_mask[:max_batch_size, :sequence_length]

         # generate max 5 tokens
         __lowercase					=				input_ids.shape[-1] + 5
         if config.eos_token_id is not None and config.pad_token_id is None:
               # hack to allow generate for models such as GPT2 as is done in `generate()`
               __lowercase					=				config.eos_token_id
         return config, input_ids, attention_mask, max_length





   @is_pt_flax_cross_test
   def 		snake_case__			(				self	:       Optional[int]   )						->					Optional[int]:
         """simple docstring"""







         __lowercase	,			__lowercase	,			__lowercase	,			__lowercase					=				self._get_input_ids_and_config()
         __lowercase					=				False
         __lowercase					=				max_length
         __lowercase					=				0

         for model_class in self.all_generative_model_classes:
               __lowercase					=				model_class(_lowerCamelCase   )

               __lowercase					=				model_class.__name__[4:]  # Skip the "Flax" at the beginning
               __lowercase					=				getattr(_lowerCamelCase			,     _lowerCamelCase   )
               __lowercase					=				pt_model_class(_lowerCamelCase   ).eval()
               __lowercase					=				load_flax_weights_in_pytorch_model(_lowerCamelCase			,     flax_model.params   )

               __lowercase					=				flax_model.generate(_lowerCamelCase   ).sequences
               __lowercase					=				pt_model.generate(torch.tensor(_lowerCamelCase			,     dtype=torch.long   )   )

               if flax_generation_outputs.shape[-1] > pt_generation_outputs.shape[-1]:
                     __lowercase					=				flax_generation_outputs[:, : pt_generation_outputs.shape[-1]]

               self.assertListEqual(pt_generation_outputs.numpy().tolist()			,     flax_generation_outputs.tolist()   )





   def 		snake_case__			(				self	:       int   )						->					int:
         """simple docstring"""







         __lowercase	,			__lowercase	,			__lowercase	,			__lowercase					=				self._get_input_ids_and_config()
         __lowercase					=				False
         __lowercase					=				max_length

         for model_class in self.all_generative_model_classes:
               __lowercase					=				model_class(_lowerCamelCase   )

               __lowercase					=				model.generate(_lowerCamelCase   ).sequences
               self.assertEqual(generation_outputs.shape[-1]			,     _lowerCamelCase   )

               __lowercase					=				jit(model.generate   )
               __lowercase					=				jit_generate(_lowerCamelCase   ).sequences

               self.assertListEqual(generation_outputs.tolist()			,     jit_generation_outputs.tolist()   )





   def 		snake_case__			(				self	:       Union[str, Any]   )						->					List[Any]:
         """simple docstring"""







         __lowercase	,			__lowercase	,			__lowercase	,			__lowercase					=				self._get_input_ids_and_config()
         __lowercase					=				True
         __lowercase					=				max_length

         for model_class in self.all_generative_model_classes:
               __lowercase					=				model_class(_lowerCamelCase   )

               __lowercase					=				model.generate(_lowerCamelCase   ).sequences
               self.assertEqual(generation_outputs.shape[-1]			,     _lowerCamelCase   )

               __lowercase					=				jit(model.generate   )
               __lowercase					=				jit_generate(_lowerCamelCase   ).sequences

               self.assertListEqual(generation_outputs.tolist()			,     jit_generation_outputs.tolist()   )





   def 		snake_case__			(				self	:       List[str]   )						->					List[str]:
         """simple docstring"""







         __lowercase	,			__lowercase	,			__lowercase	,			__lowercase					=				self._get_input_ids_and_config()
         __lowercase					=				False
         __lowercase					=				max_length
         __lowercase					=				2

         for model_class in self.all_generative_model_classes:
               __lowercase					=				model_class(_lowerCamelCase   )

               __lowercase					=				model.generate(_lowerCamelCase   ).sequences
               self.assertEqual(generation_outputs.shape[-1]			,     _lowerCamelCase   )

               __lowercase					=				jit(model.generate   )
               __lowercase					=				jit_generate(_lowerCamelCase   ).sequences

               self.assertListEqual(generation_outputs.tolist()			,     jit_generation_outputs.tolist()   )





   def 		snake_case__			(				self	:       Any   )						->					Union[str, Any]:
         """simple docstring"""







         __lowercase	,			__lowercase	,			__lowercase	,			__lowercase					=				self._get_input_ids_and_config()
         __lowercase					=				False
         __lowercase					=				max_length
         __lowercase					=				2
         __lowercase					=				2

         for model_class in self.all_generative_model_classes:
               __lowercase					=				model_class(_lowerCamelCase   )

               __lowercase					=				model.generate(_lowerCamelCase   ).sequences
               self.assertEqual(generation_outputs.shape[0]			,     input_ids.shape[0] * config.num_return_sequences   )





   def 		snake_case__			(				self	:       List[str]   )						->					str:
         """simple docstring"""







         __lowercase	,			__lowercase	,			__lowercase	,			__lowercase					=				self._get_input_ids_and_config()
         __lowercase					=				True
         __lowercase					=				max_length
         __lowercase					=				0.8
         __lowercase					=				10
         __lowercase					=				0.3
         __lowercase					=				1
         __lowercase					=				8
         __lowercase					=				9

         for model_class in self.all_generative_model_classes:
               __lowercase					=				model_class(_lowerCamelCase   )

               __lowercase					=				model.generate(_lowerCamelCase   ).sequences
               self.assertEqual(generation_outputs.shape[-1]			,     _lowerCamelCase   )

               __lowercase					=				jit(model.generate   )
               __lowercase					=				jit_generate(_lowerCamelCase   ).sequences

               self.assertListEqual(generation_outputs.tolist()			,     jit_generation_outputs.tolist()   )





   def 		snake_case__			(				self	:       Dict   )						->					Any:
         """simple docstring"""







         __lowercase	,			__lowercase	,			__lowercase	,			__lowercase					=				self._get_input_ids_and_config()
         __lowercase					=				max_length
         __lowercase					=				1
         __lowercase					=				8
         __lowercase					=				9

         for model_class in self.all_generative_model_classes:
               __lowercase					=				model_class(_lowerCamelCase   )

               __lowercase					=				model.generate(_lowerCamelCase   ).sequences
               self.assertEqual(generation_outputs.shape[-1]			,     _lowerCamelCase   )

               __lowercase					=				jit(model.generate   )
               __lowercase					=				jit_generate(_lowerCamelCase   ).sequences

               self.assertListEqual(generation_outputs.tolist()			,     jit_generation_outputs.tolist()   )





   def 		snake_case__			(				self	:       Optional[Any]   )						->					Union[str, Any]:
         """simple docstring"""







         __lowercase	,			__lowercase	,			__lowercase	,			__lowercase					=				self._get_input_ids_and_config()
         __lowercase					=				max_length
         __lowercase					=				2
         __lowercase					=				1
         __lowercase					=				8
         __lowercase					=				9

         for model_class in self.all_generative_model_classes:
               __lowercase					=				model_class(_lowerCamelCase   )

               __lowercase					=				model.generate(_lowerCamelCase   ).sequences
               self.assertEqual(generation_outputs.shape[-1]			,     _lowerCamelCase   )

               __lowercase					=				jit(model.generate   )
               __lowercase					=				jit_generate(_lowerCamelCase   ).sequences

               self.assertListEqual(generation_outputs.tolist()			,     jit_generation_outputs.tolist()   )





   def 		snake_case__			(				self	:       Dict   )						->					Tuple:
         """simple docstring"""







         __lowercase	,			__lowercase	,			__lowercase	,			__lowercase					=				self._get_input_ids_and_config()

         # pad attention mask on the left
         __lowercase					=				attention_mask.at[(0, 0)].set(0   )

         __lowercase					=				False
         __lowercase					=				max_length

         for model_class in self.all_generative_model_classes:
               __lowercase					=				model_class(_lowerCamelCase   )

               __lowercase					=				model.generate(_lowerCamelCase			,     attention_mask=_lowerCamelCase   ).sequences
               self.assertEqual(generation_outputs.shape[-1]			,     _lowerCamelCase   )

               __lowercase					=				jit(model.generate   )
               __lowercase					=				jit_generate(_lowerCamelCase			,     attention_mask=_lowerCamelCase   ).sequences

               self.assertListEqual(generation_outputs.tolist()			,     jit_generation_outputs.tolist()   )





   def 		snake_case__			(				self	:       Optional[int]   )						->					List[str]:
         """simple docstring"""







         __lowercase	,			__lowercase	,			__lowercase	,			__lowercase					=				self._get_input_ids_and_config()

         # pad attention mask on the left
         __lowercase					=				attention_mask.at[(0, 0)].set(0   )

         __lowercase					=				True
         __lowercase					=				max_length

         for model_class in self.all_generative_model_classes:
               __lowercase					=				model_class(_lowerCamelCase   )

               __lowercase					=				model.generate(_lowerCamelCase			,     attention_mask=_lowerCamelCase   ).sequences
               self.assertEqual(generation_outputs.shape[-1]			,     _lowerCamelCase   )

               __lowercase					=				jit(model.generate   )
               __lowercase					=				jit_generate(_lowerCamelCase			,     attention_mask=_lowerCamelCase   ).sequences

               self.assertListEqual(generation_outputs.tolist()			,     jit_generation_outputs.tolist()   )







   def 		snake_case__			(				self	:       Optional[Any]   )						->					List[str]:
         """simple docstring"""







         __lowercase	,			__lowercase	,			__lowercase	,			__lowercase					=				self._get_input_ids_and_config()

         # pad attention mask on the left
         __lowercase					=				attention_mask.at[(0, 0)].set(0   )

         __lowercase					=				2
         __lowercase					=				max_length

         for model_class in self.all_generative_model_classes:
               __lowercase					=				model_class(_lowerCamelCase   )

               __lowercase					=				model.generate(_lowerCamelCase			,     attention_mask=_lowerCamelCase   ).sequences
               self.assertEqual(generation_outputs.shape[-1]			,     _lowerCamelCase   )

               __lowercase					=				jit(model.generate   )
               __lowercase					=				jit_generate(_lowerCamelCase			,     attention_mask=_lowerCamelCase   ).sequences

               self.assertListEqual(generation_outputs.tolist()			,     jit_generation_outputs.tolist()   )





@require_flax
class 			_lowerCAmelCase   ( unittest.TestCase			):






   """simple docstring"""





   def 		snake_case__			(				self	:       Union[str, Any]   )						->					Optional[Any]:
         """simple docstring"""







         __lowercase					=				AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-bert"""   )
         __lowercase					=				FlaxAutoModelForCausalLM.from_pretrained("""hf-internal-testing/tiny-bert-flax-only"""   )

         __lowercase					=				"""Hello world"""
         __lowercase					=				tokenizer(_lowerCamelCase			,     return_tensors="""np"""   ).input_ids

         # typos are quickly detected (the correct argument is `do_sample`)
         with self.assertRaisesRegex(_lowerCamelCase			,     """do_samples"""   ):
               model.generate(_lowerCamelCase			,     do_samples=_lowerCamelCase   )

         # arbitrary arguments that will not be used anywhere are also not accepted
         with self.assertRaisesRegex(_lowerCamelCase			,     """foo"""   ):
               __lowercase					=				{"""foo""": """bar"""}
               model.generate(_lowerCamelCase			,     **_lowerCamelCase   )










import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 			_lowerCAmelCase   ( _UpperCAmelCase    ,					unittest.TestCase			):






			"""simple docstring"""





			lowercase__      :  Tuple      =			LxmertTokenizer
			lowercase__      :  List[str]      =			LxmertTokenizerFast
			lowercase__      :  Optional[Any]      =			True
			lowercase__      :  List[Any]      =			True





			def 		snake_case__			(				self	:       Tuple   )						->					Tuple:
									"""simple docstring"""







									super().setUp()

									__lowercase					=				[
									    """[UNK]""",
									    """[CLS]""",
									    """[SEP]""",
									    """want""",
									    """##want""",
									    """##ed""",
									    """wa""",
									    """un""",
									    """runn""",
									    """##ing""",
									    """,""",
									    """low""",
									    """lowest""",
									]
									__lowercase					=				os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["""vocab_file"""]   )
									with open(self.vocab_file			,     """w"""			,     encoding="""utf-8"""   ) as vocab_writer:
															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]   )   )





			def 		snake_case__			(				self	:       Optional[int]			,     lowercase	:       int   )						->					List[Any]:
									"""simple docstring"""







									__lowercase					=				"""UNwant\u00E9d,running"""
									__lowercase					=				"""unwanted, running"""
									return input_text, output_text





			def 		snake_case__			(				self	:       str   )						->					Any:
									"""simple docstring"""







									__lowercase					=				self.tokenizer_class(self.vocab_file   )

									__lowercase					=				tokenizer.tokenize("""UNwant\u00E9d,running"""   )
									self.assertListEqual(lowercase			,     ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]   )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase   )			,     [7, 4, 5, 10, 8, 9]   )







			def 		snake_case__			(				self	:       Union[str, Any]   )						->					int:
									"""simple docstring"""







									if not self.test_rust_tokenizer:
															return

									__lowercase					=				self.get_tokenizer()
									__lowercase					=				self.get_rust_tokenizer()

									__lowercase					=				"""I was born in 92000, and this is falsé."""

									__lowercase					=				tokenizer.tokenize(lowercase   )
									__lowercase					=				rust_tokenizer.tokenize(lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )

									__lowercase					=				tokenizer.encode(lowercase			,     add_special_tokens=lowercase   )
									__lowercase					=				rust_tokenizer.encode(lowercase			,     add_special_tokens=lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )

									__lowercase					=				self.get_rust_tokenizer()
									__lowercase					=				tokenizer.encode(lowercase   )
									__lowercase					=				rust_tokenizer.encode(lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )





