import collections
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import (
    is_pt_flax_cross_test,
    require_flax,
    require_torch,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import is_flax_available, is_torch_available, is_vision_available

from ...test_modeling_flax_common import floats_tensor, ids_tensor, random_attention_mask
from ..bert.test_modeling_flax_bert import FlaxBertModelTester
from ..clip.test_modeling_flax_clip import FlaxCLIPVisionModelTester
from ..vit.test_modeling_flax_vit import FlaxViTModelTester


if is_flax_available():
    from transformers import (
        FlaxBertModel,
        FlaxCLIPVisionModel,
        FlaxVisionTextDualEncoderModel,
        FlaxViTModel,
        VisionTextDualEncoderConfig,
        VisionTextDualEncoderProcessor,
    )
    from transformers.modeling_flax_pytorch_utils import (
        convert_pytorch_state_dict_to_flax,
        load_flax_weights_in_pytorch_model,
    )


if is_torch_available():
    import torch

    from transformers import VisionTextDualEncoderModel

if is_vision_available():
    from PIL import Image



def A__  (			lowercase:     List[str]					)				->		Any:
       if isinstance(lowercase,		collections.abc.Iterable					):
              return x
       return (x, x)



@require_flax
class  SCREAMING_SNAKE_CASE_ :
  '''simple docstring'''


  def 		SCREAMING_SNAKE_CASE_		(				self    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					int     )    ->				int:
         pass







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]     )    ->				Any:
         pass







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]     )    ->				Optional[Any]:
         pass







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]      ,	SCREAMING_SNAKE_CASE__    :					np.ndarray      ,	SCREAMING_SNAKE_CASE__    :					np.ndarray      ,	SCREAMING_SNAKE_CASE__    :					float     )    ->				List[Any]:
         A						:     int         =np.abs((a - b)     ).max()
         self.assertLessEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	f'Difference between torch and flax is {diff} (>= {tol}).'     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]      ,	SCREAMING_SNAKE_CASE__    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					List[Any]      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]=None      ,	**SCREAMING_SNAKE_CASE__    :					int     )    ->				Tuple:
         A						:     Optional[int]         =VisionTextDualEncoderConfig.from_vision_text_configs(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

         A						:     Optional[int]         =FlaxVisionTextDualEncoderModel(SCREAMING_SNAKE_CASE__     )

         A						:     Tuple         =model(input_ids=SCREAMING_SNAKE_CASE__      ,	pixel_values=SCREAMING_SNAKE_CASE__      ,	attention_mask=SCREAMING_SNAKE_CASE__     )

         self.assertEqual(output['text_embeds'].shape      ,	(input_ids.shape[0], config.projection_dim)     )
         self.assertEqual(output['image_embeds'].shape      ,	(pixel_values.shape[0], config.projection_dim)     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					int      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]=None      ,	**SCREAMING_SNAKE_CASE__    :					Union[str, Any]     )    ->				Any:
         A  ,  A						:     Union[str, Any]         =self.get_vision_text_model(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
         A						:     List[str]         ={'vision_model': vision_model, 'text_model': text_model}
         A						:     str         =FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**SCREAMING_SNAKE_CASE__     )

         A						:     Optional[int]         =model(input_ids=SCREAMING_SNAKE_CASE__      ,	pixel_values=SCREAMING_SNAKE_CASE__      ,	attention_mask=SCREAMING_SNAKE_CASE__     )

         self.assertEqual(output['text_embeds'].shape      ,	(input_ids.shape[0], model.config.projection_dim)     )
         self.assertEqual(output['image_embeds'].shape      ,	(pixel_values.shape[0], model.config.projection_dim)     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					List[str]=None      ,	**SCREAMING_SNAKE_CASE__    :					Dict     )    ->				Tuple:
         A  ,  A						:     Any         =self.get_vision_text_model(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
         A						:     List[str]         ={'vision_model': vision_model, 'text_model': text_model}
         A						:     int         =FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**SCREAMING_SNAKE_CASE__     )

         A						:     int         =model(input_ids=SCREAMING_SNAKE_CASE__      ,	pixel_values=SCREAMING_SNAKE_CASE__      ,	attention_mask=SCREAMING_SNAKE_CASE__     )
         A						:     List[Any]         =output[0]

         with tempfile.TemporaryDirectory() as tmpdirname:
                model.save_pretrained(SCREAMING_SNAKE_CASE__     )
                A						:     int         =FlaxVisionTextDualEncoderModel.from_pretrained(SCREAMING_SNAKE_CASE__     )

                A						:     Optional[int]         =model(input_ids=SCREAMING_SNAKE_CASE__      ,	pixel_values=SCREAMING_SNAKE_CASE__      ,	attention_mask=SCREAMING_SNAKE_CASE__     )
                A						:     List[Any]         =after_output[0]
                A						:     List[str]         =np.amax(np.abs(out_a - out_a     )     )
                self.assertLessEqual(SCREAMING_SNAKE_CASE__      ,	1e-3     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					List[str]      ,	SCREAMING_SNAKE_CASE__    :					List[Any]      ,	SCREAMING_SNAKE_CASE__    :					int=None      ,	**SCREAMING_SNAKE_CASE__    :					List[str]     )    ->				List[str]:
         A  ,  A						:     Optional[Any]         =self.get_vision_text_model(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
         A						:     Optional[Any]         ={'vision_model': vision_model, 'text_model': text_model}
         A						:     List[str]         =FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**SCREAMING_SNAKE_CASE__     )

         A						:     Union[str, Any]         =model(
             input_ids=SCREAMING_SNAKE_CASE__      ,	pixel_values=SCREAMING_SNAKE_CASE__      ,	attention_mask=SCREAMING_SNAKE_CASE__      ,	output_attentions=SCREAMING_SNAKE_CASE__     )

         A						:     Dict         =output.vision_model_output.attentions
         self.assertEqual(len(SCREAMING_SNAKE_CASE__     )      ,	vision_config.num_hidden_layers     )

         # in ViT, the seq_len equals the number of patches + 1 (we add 1 for the [CLS] token)
         A						:     Tuple         =to_atuple(vision_model.config.image_size     )
         A						:     int         =to_atuple(vision_model.config.patch_size     )
         A						:     List[str]         =(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
         A						:     Optional[Any]         =num_patches + 1
         self.assertEqual(vision_attentions[0].shape[-3:]      ,	(vision_config.num_attention_heads, seq_len, seq_len)     )

         A						:     List[Any]         =output.text_model_output.attentions
         self.assertEqual(len(SCREAMING_SNAKE_CASE__     )      ,	text_config.num_hidden_layers     )

         self.assertEqual(
             text_attentions[0].shape[-3:]      ,	(text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])      ,	)







  def 		SCREAMING_SNAKE_CASE_		(				self    :					str      ,	SCREAMING_SNAKE_CASE__    :					List[str]      ,	SCREAMING_SNAKE_CASE__    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					List[Any]     )    ->				Union[str, Any]:
         pt_model.to(SCREAMING_SNAKE_CASE__     )
         pt_model.eval()

         # prepare inputs
         A						:     Any         =inputs_dict
         A						:     Optional[int]         ={k: torch.tensor(v.tolist()     ) for k, v in flax_inputs.items()}

         with torch.no_grad():
                A						:     List[str]         =pt_model(**SCREAMING_SNAKE_CASE__     ).to_tuple()

         A						:     Union[str, Any]         =fx_model(**SCREAMING_SNAKE_CASE__     ).to_tuple()
         self.assertEqual(len(SCREAMING_SNAKE_CASE__     )      ,	len(SCREAMING_SNAKE_CASE__     )      ,	'Output lengths differ between Flax and PyTorch'     )
         for fx_output, pt_output in zip(fx_outputs[:4]      ,	pt_outputs[:4]     ):
                self.assert_almost_equals(SCREAMING_SNAKE_CASE__      ,	pt_output.numpy()      ,	4e-2     )

         # PT -> Flax
         with tempfile.TemporaryDirectory() as tmpdirname:
                pt_model.save_pretrained(SCREAMING_SNAKE_CASE__     )
                A						:     int         =FlaxVisionTextDualEncoderModel.from_pretrained(SCREAMING_SNAKE_CASE__      ,	from_pt=SCREAMING_SNAKE_CASE__     )

         A						:     int         =fx_model_loaded(**SCREAMING_SNAKE_CASE__     ).to_tuple()
         self.assertEqual(len(SCREAMING_SNAKE_CASE__     )      ,	len(SCREAMING_SNAKE_CASE__     )      ,	'Output lengths differ between Flax and PyTorch'     )
         for fx_output_loaded, pt_output in zip(fx_outputs_loaded[:4]      ,	pt_outputs[:4]     ):
                self.assert_almost_equals(SCREAMING_SNAKE_CASE__      ,	pt_output.numpy()      ,	4e-2     )

         # Flax -> PT
         with tempfile.TemporaryDirectory() as tmpdirname:
                fx_model.save_pretrained(SCREAMING_SNAKE_CASE__     )
                A						:     List[Any]         =VisionTextDualEncoderModel.from_pretrained(SCREAMING_SNAKE_CASE__      ,	from_flax=SCREAMING_SNAKE_CASE__     )

         pt_model_loaded.to(SCREAMING_SNAKE_CASE__     )
         pt_model_loaded.eval()

         with torch.no_grad():
                A						:     str         =pt_model_loaded(**SCREAMING_SNAKE_CASE__     ).to_tuple()

         self.assertEqual(len(SCREAMING_SNAKE_CASE__     )      ,	len(SCREAMING_SNAKE_CASE__     )      ,	'Output lengths differ between Flax and PyTorch'     )
         for fx_output, pt_output_loaded in zip(fx_outputs[:4]      ,	pt_outputs_loaded[:4]     ):
                self.assert_almost_equals(SCREAMING_SNAKE_CASE__      ,	pt_output_loaded.numpy()      ,	4e-2     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					str     )    ->				int:
         A						:     int         =VisionTextDualEncoderConfig.from_vision_text_configs(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

         A						:     int         =VisionTextDualEncoderModel(SCREAMING_SNAKE_CASE__     )
         A						:     Tuple         =FlaxVisionTextDualEncoderModel(SCREAMING_SNAKE_CASE__     )

         A						:     Any         =convert_pytorch_state_dict_to_flax(pt_model.state_dict()      ,	SCREAMING_SNAKE_CASE__     )
         A						:     Any         =fx_state

         self.check_pt_flax_equivalence(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					int      ,	SCREAMING_SNAKE_CASE__    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]     )    ->				Optional[int]:
         A						:     Any         =VisionTextDualEncoderConfig.from_vision_text_configs(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

         A						:     List[Any]         =VisionTextDualEncoderModel(SCREAMING_SNAKE_CASE__     )
         A						:     str         =FlaxVisionTextDualEncoderModel(SCREAMING_SNAKE_CASE__     )

         A						:     str         =load_flax_weights_in_pytorch_model(SCREAMING_SNAKE_CASE__      ,	fx_model.params     )

         self.check_pt_flax_equivalence(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]     )    ->				List[str]:
         A						:     Optional[Any]         =self.prepare_config_and_inputs()
         self.check_model_from_pretrained_configs(**SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				str:
         A						:     List[str]         =self.prepare_config_and_inputs()
         self.check_vision_text_dual_encoder_from_pretrained(**SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Any     )    ->				List[Any]:
         A						:     Any         =self.prepare_config_and_inputs()
         self.check_save_load(**SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]     )    ->				Optional[int]:
         A						:     Optional[int]         =self.prepare_config_and_inputs()
         self.check_vision_text_output_attention(**SCREAMING_SNAKE_CASE__     )







  @is_pt_flax_cross_test
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				Union[str, Any]:
         A						:     Optional[Any]         =self.prepare_config_and_inputs()
         A						:     Any         =config_inputs_dict.pop('vision_config'     )
         A						:     Optional[int]         =config_inputs_dict.pop('text_config'     )

         A						:     Dict         =config_inputs_dict

         self.check_equivalence_pt_to_flax(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
         self.check_equivalence_flax_to_pt(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]     )    ->				str:
         A  ,  A						:     int         =self.get_pretrained_model_and_inputs()

         A						:     Dict         =model_a(**SCREAMING_SNAKE_CASE__     )
         A						:     Optional[Any]         =outputs[0]

         with tempfile.TemporaryDirectory() as tmp_dirname:
                model_a.save_pretrained(SCREAMING_SNAKE_CASE__     )
                A						:     str         =FlaxVisionTextDualEncoderModel.from_pretrained(SCREAMING_SNAKE_CASE__     )

                A						:     Union[str, Any]         =model_a(**SCREAMING_SNAKE_CASE__     )
                A						:     Any         =after_outputs[0]
                A						:     Any         =np.amax(np.abs(out_a - out_a     )     )
                self.assertLessEqual(SCREAMING_SNAKE_CASE__      ,	1e-5     )







@require_flax
class  SCREAMING_SNAKE_CASE_ (					lowerCAmelCase_   ,						unittest.TestCase			):
  '''simple docstring'''


  def 		SCREAMING_SNAKE_CASE_		(				self    :					Any     )    ->				Tuple:
         A						:     str         =FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(
             'hf-internal-testing/tiny-random-vit'      ,	'hf-internal-testing/tiny-bert'      ,	vision_from_pt=SCREAMING_SNAKE_CASE__      ,	text_from_pt=SCREAMING_SNAKE_CASE__      ,	)
         A						:     Optional[int]         =13
         A						:     str         =floats_tensor(
             [
                 batch_size,
                 model.config.vision_config.num_channels,
                 model.config.vision_config.image_size,
                 model.config.vision_config.image_size,
             ]     )
         A						:     Dict         =ids_tensor([batch_size, 4]      ,	model.config.text_config.vocab_size     )
         A						:     List[Any]         =random_attention_mask([batch_size, 4]     )
         A						:     Any         ={'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

         return model, inputs







  def 		SCREAMING_SNAKE_CASE_		(				self    :					int      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]      ,	SCREAMING_SNAKE_CASE__    :					List[str]     )    ->				Optional[int]:
         A						:     Dict         =FlaxViTModel(SCREAMING_SNAKE_CASE__     )
         A						:     List[str]         =FlaxBertModel(SCREAMING_SNAKE_CASE__     )
         return vision_model, text_model







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]     )    ->				Dict:
         A						:     Any         =FlaxViTModelTester(self     )
         A						:     Any         =FlaxBertModelTester(self     )
         A						:     List[str]         =vit_model_tester.prepare_config_and_inputs()
         A						:     List[Any]         =bert_model_tester.prepare_config_and_inputs()

         A  ,  A						:     List[str]         =vision_config_and_inputs

         A  ,  A  ,  A  ,  A						:     Optional[int]         =text_config_and_inputs

         # make sure that cross attention layers are added
         return {
             "text_config": text_config,
             "vision_config": vision_config,
             "pixel_values": pixel_values,
             "attention_mask": attention_mask,
             "input_ids": input_ids,
             "token_type_ids": token_type_ids,
         }







@require_torch
class  SCREAMING_SNAKE_CASE_ (					lowerCAmelCase_   ,						unittest.TestCase			):
  '''simple docstring'''


  def 		SCREAMING_SNAKE_CASE_		(				self    :					int     )    ->				Any:
         A						:     Tuple         =FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(
             'hf-internal-testing/tiny-random-clip'      ,	'hf-internal-testing/tiny-bert'      ,	vision_from_pt=SCREAMING_SNAKE_CASE__      ,	text_from_pt=SCREAMING_SNAKE_CASE__      ,	)
         A						:     Any         =13
         A						:     Tuple         =floats_tensor(
             [
                 batch_size,
                 model.config.vision_config.num_channels,
                 model.config.vision_config.image_size,
                 model.config.vision_config.image_size,
             ]     )
         A						:     Optional[Any]         =ids_tensor([batch_size, 4]      ,	model.config.text_config.vocab_size     )
         A						:     List[Any]         =random_attention_mask([batch_size, 4]     )
         A						:     List[Any]         ={'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

         return model, inputs







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]     )    ->				Any:
         A						:     Optional[Any]         =FlaxCLIPVisionModel(SCREAMING_SNAKE_CASE__     )
         A						:     Union[str, Any]         =FlaxBertModel(SCREAMING_SNAKE_CASE__     )
         return vision_model, text_model







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]     )    ->				Tuple:
         A						:     Any         =FlaxCLIPVisionModelTester(self     )
         A						:     int         =FlaxBertModelTester(self     )
         A						:     Optional[int]         =clip_model_tester.prepare_config_and_inputs()
         A						:     Any         =bert_model_tester.prepare_config_and_inputs()

         A  ,  A						:     Any         =vision_config_and_inputs

         A  ,  A  ,  A  ,  A						:     List[Any]         =text_config_and_inputs

         # make sure that cross attention layers are added
         return {
             "text_config": text_config,
             "vision_config": vision_config,
             "pixel_values": pixel_values,
             "attention_mask": attention_mask,
             "input_ids": input_ids,
             "token_type_ids": token_type_ids,
         }







@require_flax
@require_vision
class  SCREAMING_SNAKE_CASE_ (					unittest.TestCase			):
  '''simple docstring'''


  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Union[str, Any]     )    ->				Any:
         A						:     Dict         =FlaxVisionTextDualEncoderModel.from_pretrained('clip-italian/clip-italian'      ,	logit_scale_init_value=1.0     )
         A						:     List[str]         =VisionTextDualEncoderProcessor.from_pretrained('clip-italian/clip-italian'     )

         A						:     Optional[Any]         =Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'     )
         A						:     int         =processor(
             text=['una foto di un gatto', 'una foto di un cane']      ,	images=SCREAMING_SNAKE_CASE__      ,	padding=SCREAMING_SNAKE_CASE__      ,	return_tensors='np'     )

         A						:     Tuple         =model(**SCREAMING_SNAKE_CASE__     )

         # verify the logits
         self.assertEqual(outputs.logits_per_image.shape      ,	(inputs.pixel_values.shape[0], inputs.input_ids.shape[0])     )
         self.assertEqual(
             outputs.logits_per_text.shape      ,	(inputs.input_ids.shape[0], inputs.pixel_values.shape[0])      ,	)

         A						:     List[str]         =np.array([[1.2_2_8_4_7_2_7, 0.3_1_0_4_1_2_2]]     )

         self.assertTrue(np.allclose(outputs.logits_per_image      ,	SCREAMING_SNAKE_CASE__      ,	atol=1e-3     )     )







import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_lowercase							:      Optional[int]      							=get_tests_dir('''fixtures/test_sentencepiece.model''')



@require_sentencepiece
@require_tokenizers
class  SCREAMING_SNAKE_CASE_ (					lowerCAmelCase_   ,						unittest.TestCase			):
  '''simple docstring'''


  lowercase		:     List[str]            =				XLMRobertaTokenizer
  lowercase		:     Dict            =				XLMRobertaTokenizerFast
  lowercase		:     str            =				True
  lowercase		:     Tuple            =				True







  def 		SCREAMING_SNAKE_CASE_		(				self    :					int     )    ->				Optional[Any]:
         super().setUp()

         # We have a SentencePiece fixture for testing
         A						:     List[str]         =XLMRobertaTokenizer(SCREAMING_SNAKE_CASE__      ,	keep_accents=SCREAMING_SNAKE_CASE__     )
         tokenizer.save_pretrained(self.tmpdirname     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					int     )    ->				List[Any]:
         A						:     List[str]         ='<pad>'
         A						:     int         =1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE__     )      ,	SCREAMING_SNAKE_CASE__     )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE__     )      ,	SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]     )    ->				Any:
         A						:     List[str]         =list(self.get_tokenizer().get_vocab().keys()     )

         self.assertEqual(vocab_keys[0]      ,	'<s>'     )
         self.assertEqual(vocab_keys[1]      ,	'<pad>'     )
         self.assertEqual(vocab_keys[-1]      ,	'<mask>'     )
         self.assertEqual(len(SCREAMING_SNAKE_CASE__     )      ,	10_02     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Tuple     )    ->				Dict:
         self.assertEqual(self.get_tokenizer().vocab_size      ,	10_02     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]     )    ->				str:
         A						:     Union[str, Any]         =XLMRobertaTokenizer(SCREAMING_SNAKE_CASE__      ,	keep_accents=SCREAMING_SNAKE_CASE__     )

         A						:     str         =tokenizer.tokenize('This is a test'     )
         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	['▁This', '▁is', '▁a', '▁t', 'est']     )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__     )      ,	[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]      ,	)

         A						:     Any         =tokenizer.tokenize('I was born in 92000, and this is falsé.'     )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE__      ,	[
                 SPIECE_UNDERLINE + 'I',
                 SPIECE_UNDERLINE + 'was',
                 SPIECE_UNDERLINE + 'b',
                 'or',
                 'n',
                 SPIECE_UNDERLINE + 'in',
                 SPIECE_UNDERLINE + '',
                 '9',
                 '2',
                 '0',
                 '0',
                 '0',
                 ',',
                 SPIECE_UNDERLINE + 'and',
                 SPIECE_UNDERLINE + 'this',
                 SPIECE_UNDERLINE + 'is',
                 SPIECE_UNDERLINE + 'f',
                 'al',
                 's',
                 'é',
                 '.',
             ]      ,	)
         A						:     Tuple         =tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE__      ,	[
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
             ]      ,	)

         A						:     Union[str, Any]         =tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE__      ,	[
                 SPIECE_UNDERLINE + 'I',
                 SPIECE_UNDERLINE + 'was',
                 SPIECE_UNDERLINE + 'b',
                 'or',
                 'n',
                 SPIECE_UNDERLINE + 'in',
                 SPIECE_UNDERLINE + '',
                 '<unk>',
                 '2',
                 '0',
                 '0',
                 '0',
                 ',',
                 SPIECE_UNDERLINE + 'and',
                 SPIECE_UNDERLINE + 'this',
                 SPIECE_UNDERLINE + 'is',
                 SPIECE_UNDERLINE + 'f',
                 'al',
                 's',
                 '<unk>',
                 '.',
             ]      ,	)







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Any     )    ->				Optional[int]:
         if not self.test_slow_tokenizer:
                # as we don't have a slow version, we can't compare the outputs between slow and fast versions
                return

         A						:     Any         =(self.rust_tokenizer_class, 'hf-internal-testing/tiny-xlm-roberta', {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
                       A						:     List[Any]         =self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__      ,	**SCREAMING_SNAKE_CASE__     )
                       A						:     Dict         =self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__      ,	**SCREAMING_SNAKE_CASE__     )

                       A						:     str         =tempfile.mkdtemp()

                       A						:     Optional[int]         =tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__     )
                       A						:     Optional[Any]         =tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__     )

                       # Checks it save with the same files + the tokenizer.json file for the fast one
                       self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files     )     )
                       A						:     List[str]         =tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f     )
                       self.assertSequenceEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

                       # Checks everything loads correctly in the same way
                       A						:     Tuple         =tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__     )
                       A						:     Dict         =tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__     )

                       # Check special tokens are set accordingly on Rust and Python
                       for key in tokenizer_pp.special_tokens_map:
                              self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )     )
                              # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
                              # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

                       shutil.rmtree(SCREAMING_SNAKE_CASE__     )

                       # Save tokenizer rust, legacy_format=True
                       A						:     Optional[int]         =tempfile.mkdtemp()

                       A						:     Optional[int]         =tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__      ,	legacy_format=SCREAMING_SNAKE_CASE__     )
                       A						:     List[Any]         =tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__     )

                       # Checks it save with the same files
                       self.assertSequenceEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

                       # Checks everything loads correctly in the same way
                       A						:     Tuple         =tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__     )
                       A						:     Tuple         =tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__     )

                       # Check special tokens are set accordingly on Rust and Python
                       for key in tokenizer_pp.special_tokens_map:
                              self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )     )

                       shutil.rmtree(SCREAMING_SNAKE_CASE__     )

                       # Save tokenizer rust, legacy_format=False
                       A						:     List[Any]         =tempfile.mkdtemp()

                       A						:     Optional[int]         =tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__      ,	legacy_format=SCREAMING_SNAKE_CASE__     )
                       A						:     str         =tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__     )

                       # Checks it saved the tokenizer.json file
                       self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files     )     )

                       # Checks everything loads correctly in the same way
                       A						:     List[Any]         =tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__     )
                       A						:     List[Any]         =tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__     )

                       # Check special tokens are set accordingly on Rust and Python
                       for key in tokenizer_pp.special_tokens_map:
                              self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )     )

                       shutil.rmtree(SCREAMING_SNAKE_CASE__     )







  @cached_property
  def 		SCREAMING_SNAKE_CASE_		(				self    :					str     )    ->				Optional[int]:
         return XLMRobertaTokenizer.from_pretrained('xlm-roberta-base'     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]     )    ->				Any:
         with tempfile.NamedTemporaryFile() as f:
                shutil.copyfile(SCREAMING_SNAKE_CASE__      ,	f.name     )
                A						:     Optional[Any]         =XLMRobertaTokenizer(f.name      ,	keep_accents=SCREAMING_SNAKE_CASE__     )
                A						:     int         =pickle.dumps(SCREAMING_SNAKE_CASE__     )
         pickle.loads(SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]     )    ->				Union[str, Any]:
         if not self.test_rust_tokenizer:
                return

         A						:     Union[str, Any]         =self.get_tokenizer()
         A						:     int         =self.get_rust_tokenizer()

         A						:     List[str]         ='I was born in 92000, and this is falsé.'

         A						:     Union[str, Any]         =tokenizer.tokenize(SCREAMING_SNAKE_CASE__     )
         A						:     Optional[int]         =rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

         A						:     Any         =tokenizer.encode(SCREAMING_SNAKE_CASE__      ,	add_special_tokens=SCREAMING_SNAKE_CASE__     )
         A						:     Tuple         =rust_tokenizer.encode(SCREAMING_SNAKE_CASE__      ,	add_special_tokens=SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

         A						:     Optional[Any]         =self.get_rust_tokenizer()
         A						:     int         =tokenizer.encode(SCREAMING_SNAKE_CASE__     )
         A						:     Dict         =rust_tokenizer.encode(SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				List[str]:
         A						:     Any         ='Hello World!'
         A						:     Optional[Any]         =[0, 3_53_78, 66_61, 38, 2]
         # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
         # xlmr.eval()
         # xlmr.encode(symbols)

         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	self.big_tokenizer.encode(SCREAMING_SNAKE_CASE__     )     )







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]     )    ->				str:
         A						:     Any         =(
             'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
             ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
         )
         A						:     int         =[
             0,
             32_93,
             83,
             10,
             45_52,
             49_89,
             79_86,
             6_78,
             10,
             59_15,
             1_11,
             17_94_59,
             12_48_50,
             4,
             60_44,
             2_37,
             12,
             6,
             5,
             6,
             4,
             67_80,
             7_05,
             15,
             13_88,
             44,
             3_78,
             1_01_14,
             7_11,
             1_52,
             20,
             6,
             5,
             2_23_76,
             6_42,
             12_21,
             1_51_90,
             3_41_53,
             4_50,
             56_08,
             9_59,
             11_19,
             5_77_02,
             1_36,
             1_86,
             47,
             10_98,
             2_93_67,
             47,
             # 4426, # What fairseq tokenizes from "<unk>": "_<"
             # 3678, # What fairseq tokenizes from "<unk>": "unk"
             # 2740, # What fairseq tokenizes from "<unk>": ">"
             3,  # What we tokenize from "<unk>": "<unk>"
             6,  # Residue from the tokenization: an extra sentencepiece underline
             4,
             60_44,
             2_37,
             62_84,
             5_09_01,
             5_28,
             31,
             90,
             34,
             9_27,
             2,
         ]
         # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
         # xlmr.eval()
         # xlmr.encode(symbols)

         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	self.big_tokenizer.encode(SCREAMING_SNAKE_CASE__     )     )







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Any     )    ->				Any:
         # fmt: off
         A						:     List[Any]         ={'input_ids': [[0, 1_10_62, 8_27_72, 7, 15, 8_27_72, 5_38, 5_15_29, 2_37, 1_71_98, 12_90, 2_06, 9, 21_51_75, 13_14, 1_36, 1_71_98, 12_90, 2_06, 9, 5_63_59, 42, 12_20_09, 9, 1_64_66, 16, 8_73_44, 45_37, 9, 47_17, 7_83_81, 6, 15_99_58, 7, 15, 2_44_80, 6_18, 4, 5_27, 2_26_93, 54_28, 4, 27_77, 2_44_80, 98_74, 4, 4_35_23, 5_94, 4, 8_03, 1_83_92, 3_31_89, 18, 4, 4_35_23, 2_44_47, 1_23_99, 1_00, 2_49_55, 8_36_58, 96_26, 14_40_57, 15, 8_39, 2_23_35, 16, 1_36, 2_49_55, 8_36_58, 8_34_79, 15, 3_91_02, 7_24, 16, 6_78, 6_45, 27_89, 13_28, 45_89, 42, 12_20_09, 11_57_74, 23, 8_05, 13_28, 4_68_76, 7, 1_36, 5_38_94, 19_40, 4_22_27, 4_11_59, 1_77_21, 8_23, 4_25, 4, 2_75_12, 9_87_22, 2_06, 1_36, 55_31, 49_70, 9_19, 1_73_36, 5, 2], [0, 2_00_80, 6_18, 83, 8_27_75, 47, 4_79, 9, 15_17, 73, 5_38_94, 3_33, 8_05_81, 11_01_17, 1_88_11, 52_56, 12_95, 51, 15_25_26, 2_97, 79_86, 3_90, 12_44_16, 5_38, 3_54_31, 2_14, 98, 1_50_44, 2_57_37, 1_36, 71_08, 4_37_01, 23, 7_56, 13_53_55, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 5_81, 6_37_73, 11_94_55, 6, 14_77_97, 8_82_03, 7, 6_45, 70, 21, 32_85, 1_02_69, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=SCREAMING_SNAKE_CASE__      ,	model_name='xlm-roberta-base'      ,	revision='d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3'      ,	)






