
import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowercase__	:Any										=					get_tests_dir("fixtures/test_sentencepiece.model")







@require_sentencepiece
@require_tokenizers
class        lowercase	(      __lowerCAmelCase ,						unittest.TestCase	):
       lowercase_ : Any       =XLMRobertaTokenizer
       lowercase_ : Optional[int]       =XLMRobertaTokenizerFast
       lowercase_ : Optional[int]       =True
       lowercase_ : List[Any]       =True




       def 				A__   (       self):
            super().setUp()

            # We have a SentencePiece fixture for testing
            lowercase					=		XLMRobertaTokenizer(_UpperCamelCase     ,keep_accents=_UpperCamelCase)
            tokenizer.save_pretrained(self.tmpdirname)




       def 				A__   (       self):
            lowercase					=		"""<pad>"""
            lowercase					=		1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCamelCase)     ,_UpperCamelCase)
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCamelCase)     ,_UpperCamelCase)




       def 				A__   (       self):
            lowercase					=		list(self.get_tokenizer().get_vocab().keys())

            self.assertEqual(vocab_keys[0]     ,'''<s>''')
            self.assertEqual(vocab_keys[1]     ,'''<pad>''')
            self.assertEqual(vocab_keys[-1]     ,'''<mask>''')
            self.assertEqual(len(_UpperCamelCase)     ,1_0_0_2)




       def 				A__   (       self):
            self.assertEqual(self.get_tokenizer().vocab_size     ,1_0_0_2)




       def 				A__   (       self):
            lowercase					=		XLMRobertaTokenizer(_UpperCamelCase     ,keep_accents=_UpperCamelCase)

            lowercase					=		tokenizer.tokenize('''This is a test''')
            self.assertListEqual(_UpperCamelCase     ,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(_UpperCamelCase)     ,[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]     ,)

            lowercase					=		tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
            self.assertListEqual(
                _UpperCamelCase     ,[
                    SPIECE_UNDERLINE + '''I''',
                    SPIECE_UNDERLINE + '''was''',
                    SPIECE_UNDERLINE + '''b''',
                    '''or''',
                    '''n''',
                    SPIECE_UNDERLINE + '''in''',
                    SPIECE_UNDERLINE + '''''',
                    '''9''',
                    '''2''',
                    '''0''',
                    '''0''',
                    '''0''',
                    ''',''',
                    SPIECE_UNDERLINE + '''and''',
                    SPIECE_UNDERLINE + '''this''',
                    SPIECE_UNDERLINE + '''is''',
                    SPIECE_UNDERLINE + '''f''',
                    '''al''',
                    '''s''',
                    '''é''',
                    '''.''',
                ]     ,)
            lowercase					=		tokenizer.convert_tokens_to_ids(_UpperCamelCase)
            self.assertListEqual(
                _UpperCamelCase     ,[
                    value + tokenizer.fairseq_offset
                    for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
                    #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
                ]     ,)

            lowercase					=		tokenizer.convert_ids_to_tokens(_UpperCamelCase)
            self.assertListEqual(
                _UpperCamelCase     ,[
                    SPIECE_UNDERLINE + '''I''',
                    SPIECE_UNDERLINE + '''was''',
                    SPIECE_UNDERLINE + '''b''',
                    '''or''',
                    '''n''',
                    SPIECE_UNDERLINE + '''in''',
                    SPIECE_UNDERLINE + '''''',
                    '''<unk>''',
                    '''2''',
                    '''0''',
                    '''0''',
                    '''0''',
                    ''',''',
                    SPIECE_UNDERLINE + '''and''',
                    SPIECE_UNDERLINE + '''this''',
                    SPIECE_UNDERLINE + '''is''',
                    SPIECE_UNDERLINE + '''f''',
                    '''al''',
                    '''s''',
                    '''<unk>''',
                    '''.''',
                ]     ,)




       def 				A__   (       self):


            if not self.test_slow_tokenizer:
                 # as we don't have a slow version, we can't compare the outputs between slow and fast versions
                 return

            lowercase					=		(self.rust_tokenizer_class, """hf-internal-testing/tiny-xlm-roberta""", {})
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
                      lowercase					=		self.rust_tokenizer_class.from_pretrained(_UpperCamelCase     ,**_UpperCamelCase)
                      lowercase					=		self.tokenizer_class.from_pretrained(_UpperCamelCase     ,**_UpperCamelCase)

                      lowercase					=		tempfile.mkdtemp()

                      lowercase					=		tokenizer_r.save_pretrained(_UpperCamelCase)
                      lowercase					=		tokenizer_p.save_pretrained(_UpperCamelCase)

                      # Checks it save with the same files + the tokenizer.json file for the fast one
                      self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files))
                      lowercase					=		tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f)
                      self.assertSequenceEqual(_UpperCamelCase     ,_UpperCamelCase)

                      # Checks everything loads correctly in the same way
                      lowercase					=		tokenizer_r.from_pretrained(_UpperCamelCase)
                      lowercase					=		tokenizer_p.from_pretrained(_UpperCamelCase)

                      # Check special tokens are set accordingly on Rust and Python
                      for key in tokenizer_pp.special_tokens_map:
                           self.assertTrue(hasattr(_UpperCamelCase     ,_UpperCamelCase))
                           # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
                           # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

                      shutil.rmtree(_UpperCamelCase)

                      # Save tokenizer rust, legacy_format=True
                      lowercase					=		tempfile.mkdtemp()

                      lowercase					=		tokenizer_r.save_pretrained(_UpperCamelCase     ,legacy_format=_UpperCamelCase)
                      lowercase					=		tokenizer_p.save_pretrained(_UpperCamelCase)

                      # Checks it save with the same files
                      self.assertSequenceEqual(_UpperCamelCase     ,_UpperCamelCase)

                      # Checks everything loads correctly in the same way
                      lowercase					=		tokenizer_r.from_pretrained(_UpperCamelCase)
                      lowercase					=		tokenizer_p.from_pretrained(_UpperCamelCase)

                      # Check special tokens are set accordingly on Rust and Python
                      for key in tokenizer_pp.special_tokens_map:
                           self.assertTrue(hasattr(_UpperCamelCase     ,_UpperCamelCase))

                      shutil.rmtree(_UpperCamelCase)

                      # Save tokenizer rust, legacy_format=False
                      lowercase					=		tempfile.mkdtemp()

                      lowercase					=		tokenizer_r.save_pretrained(_UpperCamelCase     ,legacy_format=_UpperCamelCase)
                      lowercase					=		tokenizer_p.save_pretrained(_UpperCamelCase)

                      # Checks it saved the tokenizer.json file
                      self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files))

                      # Checks everything loads correctly in the same way
                      lowercase					=		tokenizer_r.from_pretrained(_UpperCamelCase)
                      lowercase					=		tokenizer_p.from_pretrained(_UpperCamelCase)

                      # Check special tokens are set accordingly on Rust and Python
                      for key in tokenizer_pp.special_tokens_map:
                           self.assertTrue(hasattr(_UpperCamelCase     ,_UpperCamelCase))

                      shutil.rmtree(_UpperCamelCase)




       @cached_property
       def 				A__   (       self):
            return XLMRobertaTokenizer.from_pretrained('''xlm-roberta-base''')




       def 				A__   (       self):


            with tempfile.NamedTemporaryFile() as f:
                 shutil.copyfile(_UpperCamelCase     ,f.name)
                 lowercase					=		XLMRobertaTokenizer(f.name     ,keep_accents=_UpperCamelCase)
                 lowercase					=		pickle.dumps(_UpperCamelCase)
            pickle.loads(_UpperCamelCase)




       def 				A__   (       self):


            if not self.test_rust_tokenizer:
                 return

            lowercase					=		self.get_tokenizer()
            lowercase					=		self.get_rust_tokenizer()

            lowercase					=		"""I was born in 92000, and this is falsé."""

            lowercase					=		tokenizer.tokenize(_UpperCamelCase)
            lowercase					=		rust_tokenizer.tokenize(_UpperCamelCase)
            self.assertListEqual(_UpperCamelCase     ,_UpperCamelCase)

            lowercase					=		tokenizer.encode(_UpperCamelCase     ,add_special_tokens=_UpperCamelCase)
            lowercase					=		rust_tokenizer.encode(_UpperCamelCase     ,add_special_tokens=_UpperCamelCase)
            self.assertListEqual(_UpperCamelCase     ,_UpperCamelCase)

            lowercase					=		self.get_rust_tokenizer()
            lowercase					=		tokenizer.encode(_UpperCamelCase)
            lowercase					=		rust_tokenizer.encode(_UpperCamelCase)
            self.assertListEqual(_UpperCamelCase     ,_UpperCamelCase)




       @slow
       def 				A__   (       self):
            lowercase					=		"""Hello World!"""
            lowercase					=		[0, 3_5_3_7_8, 6_6_6_1, 3_8, 2]
            # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
            # xlmr.eval()
            # xlmr.encode(symbols)

            self.assertListEqual(_UpperCamelCase     ,self.big_tokenizer.encode(_UpperCamelCase))




       @slow
       def 				A__   (       self):
            lowercase					=		(
                """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
                """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
            )
            lowercase					=		[
                0,
                3_2_9_3,
                8_3,
                1_0,
                4_5_5_2,
                4_9_8_9,
                7_9_8_6,
                6_7_8,
                1_0,
                5_9_1_5,
                1_1_1,
                1_7_9_4_5_9,
                1_2_4_8_5_0,
                4,
                6_0_4_4,
                2_3_7,
                1_2,
                6,
                5,
                6,
                4,
                6_7_8_0,
                7_0_5,
                1_5,
                1_3_8_8,
                4_4,
                3_7_8,
                1_0_1_1_4,
                7_1_1,
                1_5_2,
                2_0,
                6,
                5,
                2_2_3_7_6,
                6_4_2,
                1_2_2_1,
                1_5_1_9_0,
                3_4_1_5_3,
                4_5_0,
                5_6_0_8,
                9_5_9,
                1_1_1_9,
                5_7_7_0_2,
                1_3_6,
                1_8_6,
                4_7,
                1_0_9_8,
                2_9_3_6_7,
                4_7,
                # 4426, # What fairseq tokenizes from "<unk>": "_<"
                # 3678, # What fairseq tokenizes from "<unk>": "unk"
                # 2740, # What fairseq tokenizes from "<unk>": ">"
                3,  # What we tokenize from "<unk>": "<unk>"
                6,  # Residue from the tokenization: an extra sentencepiece underline
                4,
                6_0_4_4,
                2_3_7,
                6_2_8_4,
                5_0_9_0_1,
                5_2_8,
                3_1,
                9_0,
                3_4,
                9_2_7,
                2,
            ]
            # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
            # xlmr.eval()
            # xlmr.encode(symbols)

            self.assertListEqual(_UpperCamelCase     ,self.big_tokenizer.encode(_UpperCamelCase))





       @slow
       def 				A__   (       self):
            lowercase					=		{"""input_ids""": [[0, 1_1_0_6_2, 8_2_7_7_2, 7, 1_5, 8_2_7_7_2, 5_3_8, 5_1_5_2_9, 2_3_7, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 2_1_5_1_7_5, 1_3_1_4, 1_3_6, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 5_6_3_5_9, 4_2, 1_2_2_0_0_9, 9, 1_6_4_6_6, 1_6, 8_7_3_4_4, 4_5_3_7, 9, 4_7_1_7, 7_8_3_8_1, 6, 1_5_9_9_5_8, 7, 1_5, 2_4_4_8_0, 6_1_8, 4, 5_2_7, 2_2_6_9_3, 5_4_2_8, 4, 2_7_7_7, 2_4_4_8_0, 9_8_7_4, 4, 4_3_5_2_3, 5_9_4, 4, 8_0_3, 1_8_3_9_2, 3_3_1_8_9, 1_8, 4, 4_3_5_2_3, 2_4_4_4_7, 1_2_3_9_9, 1_0_0, 2_4_9_5_5, 8_3_6_5_8, 9_6_2_6, 1_4_4_0_5_7, 1_5, 8_3_9, 2_2_3_3_5, 1_6, 1_3_6, 2_4_9_5_5, 8_3_6_5_8, 8_3_4_7_9, 1_5, 3_9_1_0_2, 7_2_4, 1_6, 6_7_8, 6_4_5, 2_7_8_9, 1_3_2_8, 4_5_8_9, 4_2, 1_2_2_0_0_9, 1_1_5_7_7_4, 2_3, 8_0_5, 1_3_2_8, 4_6_8_7_6, 7, 1_3_6, 5_3_8_9_4, 1_9_4_0, 4_2_2_2_7, 4_1_1_5_9, 1_7_7_2_1, 8_2_3, 4_2_5, 4, 2_7_5_1_2, 9_8_7_2_2, 2_0_6, 1_3_6, 5_5_3_1, 4_9_7_0, 9_1_9, 1_7_3_3_6, 5, 2], [0, 2_0_0_8_0, 6_1_8, 8_3, 8_2_7_7_5, 4_7, 4_7_9, 9, 1_5_1_7, 7_3, 5_3_8_9_4, 3_3_3, 8_0_5_8_1, 1_1_0_1_1_7, 1_8_8_1_1, 5_2_5_6, 1_2_9_5, 5_1, 1_5_2_5_2_6, 2_9_7, 7_9_8_6, 3_9_0, 1_2_4_4_1_6, 5_3_8, 3_5_4_3_1, 2_1_4, 9_8, 1_5_0_4_4, 2_5_7_3_7, 1_3_6, 7_1_0_8, 4_3_7_0_1, 2_3, 7_5_6, 1_3_5_3_5_5, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 5_8_1, 6_3_7_7_3, 1_1_9_4_5_5, 6, 1_4_7_7_9_7, 8_8_2_0_3, 7, 6_4_5, 7_0, 2_1, 3_2_8_5, 1_0_2_6_9, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
            # fmt: on

            self.tokenizer_integration_test_util(
                expected_encoding=_UpperCamelCase     ,model_name='''xlm-roberta-base'''     ,revision='''d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3'''     ,)






import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
							import torch_xla.core.xla_model as xm
							import torch_xla.debug.metrics as met







class        lowercase	(      SCREAMING_SNAKE_CASE__	):




							def __init__(       self     ,*A__     ,A__=None     ,A__=None     ,**A__):
												super().__init__(*A__     ,**A__)
												lowercase					=		eval_examples
												lowercase					=		post_process_function




							def 				A__   (       self     ,A__ = None     ,A__=None     ,A__ = None     ,A__ = "eval"     ,**A__     ,):
												lowercase					=		gen_kwargs.copy()
												lowercase					=		(
												    gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length''') is not None else self.args.generation_max_length
												)
												lowercase					=		(
												    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams''') is not None else self.args.generation_num_beams
												)
												lowercase					=		gen_kwargs

												lowercase					=		self.eval_dataset if eval_dataset is None else eval_dataset
												lowercase					=		self.get_eval_dataloader(A__)
												lowercase					=		self.eval_examples if eval_examples is None else eval_examples

												# Temporarily disable metric computation, we will do it in the loop here.
												lowercase					=		self.compute_metrics
												lowercase					=		None
												lowercase					=		time.time()
												lowercase					=		self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	lowercase					=		eval_loop(
																	    A__     ,description='''Evaluation'''     ,prediction_loss_only=True if compute_metrics is None else None     ,ignore_keys=A__     ,metric_key_prefix=A__     ,)
												finally:
																	lowercase					=		compute_metrics
												lowercase					=		self.args.eval_batch_size * self.args.world_size
												if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																	start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
												output.metrics.update(
												    speed_metrics(
												        A__     ,A__     ,num_samples=output.num_samples     ,num_steps=math.ceil(output.num_samples / total_batch_size)     ,))

												if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																	# Only the main node write the results by default
																	lowercase					=		self.post_process_function(A__     ,A__     ,A__)
																	lowercase					=		self.compute_metrics(A__)

																	# Prefix all keys with metric_key_prefix + '_'
																	for key in list(metrics.keys()):
																						if not key.startswith(f'{metric_key_prefix}_'):
																											lowercase					=		metrics.pop(A__)

																	metrics.update(output.metrics)
												else:
																	lowercase					=		output.metrics

												if self.args.should_log:
																	# Only the main node log the results by default
																	self.log(A__)

												if self.args.tpu_metrics_debug or self.args.debug:
																	# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																	xm.master_print(met.metrics_report())

												lowercase					=		self.callback_handler.on_evaluate(self.args     ,self.state     ,self.control     ,A__)
												return metrics





							def 				A__   (       self     ,A__     ,A__     ,A__=None     ,A__ = "test"     ,**A__):
												lowercase					=		gen_kwargs.copy()

												lowercase					=		self.get_test_dataloader(A__)

												# Temporarily disable metric computation, we will do it in the loop here.
												lowercase					=		self.compute_metrics
												lowercase					=		None
												lowercase					=		time.time()
												lowercase					=		self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	lowercase					=		eval_loop(
																	    A__     ,description='''Prediction'''     ,prediction_loss_only=True if compute_metrics is None else None     ,ignore_keys=A__     ,metric_key_prefix=A__     ,)
												finally:
																	lowercase					=		compute_metrics

												lowercase					=		self.args.eval_batch_size * self.args.world_size
												if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																	start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
												output.metrics.update(
												    speed_metrics(
												        A__     ,A__     ,num_samples=output.num_samples     ,num_steps=math.ceil(output.num_samples / total_batch_size)     ,))
												if self.post_process_function is None or self.compute_metrics is None:
																	return output

												lowercase					=		self.post_process_function(A__     ,A__     ,A__     ,'''predict''')
												lowercase					=		self.compute_metrics(A__)

												# Prefix all keys with metric_key_prefix + '_'
												for key in list(metrics.keys()):
																	if not key.startswith(f'{metric_key_prefix}_'):
																						lowercase					=		metrics.pop(A__)
												metrics.update(output.metrics)
												return PredictionOutput(predictions=predictions.predictions     ,label_ids=predictions.label_ids     ,metrics=A__)




