


"""simple docstring"""


from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
   import tensorflow as tf

   from transformers import (
       TFRoFormerForCausalLM,
       TFRoFormerForMaskedLM,
       TFRoFormerForMultipleChoice,
       TFRoFormerForQuestionAnswering,
       TFRoFormerForSequenceClassification,
       TFRoFormerForTokenClassification,
       TFRoFormerModel,
   )
   from transformers.models.roformer.modeling_tf_roformer import (
       TFRoFormerSelfAttention,
       TFRoFormerSinusoidalPositionalEmbedding,
   )



class   _a      :
    def __init__(   self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]							,					SCREAMING_SNAKE_CASE__							:					List[Any]=13							,					SCREAMING_SNAKE_CASE__							:					Dict=7							,					SCREAMING_SNAKE_CASE__							:					str=True							,					SCREAMING_SNAKE_CASE__							:					str=True							,					SCREAMING_SNAKE_CASE__							:					Tuple=True							,					SCREAMING_SNAKE_CASE__							:					List[Any]=True							,					SCREAMING_SNAKE_CASE__							:					Tuple=99							,					SCREAMING_SNAKE_CASE__							:					str=32							,					SCREAMING_SNAKE_CASE__							:					Tuple=2							,					SCREAMING_SNAKE_CASE__							:					int=4							,					SCREAMING_SNAKE_CASE__							:					Dict=37							,					SCREAMING_SNAKE_CASE__							:					Optional[int]="gelu"							,					SCREAMING_SNAKE_CASE__							:					Tuple=0.1							,					SCREAMING_SNAKE_CASE__							:					List[Any]=0.1							,					SCREAMING_SNAKE_CASE__							:					List[Any]=5_12							,					SCREAMING_SNAKE_CASE__							:					Optional[int]=16							,					SCREAMING_SNAKE_CASE__							:					str=2							,					SCREAMING_SNAKE_CASE__							:					Dict=0.02							,					SCREAMING_SNAKE_CASE__							:					List[str]=3							,					SCREAMING_SNAKE_CASE__							:					int=4							,					SCREAMING_SNAKE_CASE__							:					Optional[Any]=None							,					):
          lowerCamelCase__           =		parent
          lowerCamelCase__           =		13
          lowerCamelCase__           =		7
          lowerCamelCase__           =		True
          lowerCamelCase__           =		True
          lowerCamelCase__           =		True
          lowerCamelCase__           =		True
          lowerCamelCase__           =		99
          lowerCamelCase__           =		32
          lowerCamelCase__           =		2
          lowerCamelCase__           =		4
          lowerCamelCase__           =		37
          lowerCamelCase__           =		'gelu'
          lowerCamelCase__           =		0.1
          lowerCamelCase__           =		0.1
          lowerCamelCase__           =		5_12
          lowerCamelCase__           =		16
          lowerCamelCase__           =		2
          lowerCamelCase__           =		0.02
          lowerCamelCase__           =		3
          lowerCamelCase__           =		4
          lowerCamelCase__           =		None
    def      _UpperCamelCase       (   self							:					Optional[int]				):
          lowerCamelCase__           =		ids_tensor([self.batch_size, self.seq_length]							,					self.vocab_size				)

          lowerCamelCase__           =		None
          if self.use_input_mask:
                lowerCamelCase__           =		random_attention_mask([self.batch_size, self.seq_length]				)

          lowerCamelCase__           =		None
          if self.use_token_type_ids:
                lowerCamelCase__           =		ids_tensor([self.batch_size, self.seq_length]							,					self.type_vocab_size				)

          lowerCamelCase__           =		None
          lowerCamelCase__           =		None
          lowerCamelCase__           =		None
          if self.use_labels:
                lowerCamelCase__           =		ids_tensor([self.batch_size]							,					self.type_sequence_label_size				)
                lowerCamelCase__           =		ids_tensor([self.batch_size, self.seq_length]							,					self.num_labels				)
                lowerCamelCase__           =		ids_tensor([self.batch_size]							,					self.num_choices				)

          lowerCamelCase__           =		RoFormerConfig(
              vocab_size=self.vocab_size							,					hidden_size=self.hidden_size							,					num_hidden_layers=self.num_hidden_layers							,					num_attention_heads=self.num_attention_heads							,					intermediate_size=self.intermediate_size							,					hidden_act=self.hidden_act							,					hidden_dropout_prob=self.hidden_dropout_prob							,					attention_probs_dropout_prob=self.attention_probs_dropout_prob							,					max_position_embeddings=self.max_position_embeddings							,					type_vocab_size=self.type_vocab_size							,					initializer_range=self.initializer_range							,					return_dict=SCREAMING_SNAKE_CASE__							,					)

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    def      _UpperCamelCase       (   self							:					str							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]							,					SCREAMING_SNAKE_CASE__							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					Any							,					SCREAMING_SNAKE_CASE__							:					str							,					SCREAMING_SNAKE_CASE__							:					int							,					SCREAMING_SNAKE_CASE__							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					Tuple				):
          lowerCamelCase__           =		TFRoFormerModel(config=SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

          lowerCamelCase__           =		[input_ids, input_mask]
          lowerCamelCase__           =		model(SCREAMING_SNAKE_CASE__				)

          lowerCamelCase__           =		model(SCREAMING_SNAKE_CASE__				)

          self.parent.assertEqual(result.last_hidden_state.shape							,					(self.batch_size, self.seq_length, self.hidden_size)				)
    def      _UpperCamelCase       (   self							:					str							,					SCREAMING_SNAKE_CASE__							:					Optional[int]							,					SCREAMING_SNAKE_CASE__							:					Optional[Any]							,					SCREAMING_SNAKE_CASE__							:					Optional[Any]							,					SCREAMING_SNAKE_CASE__							:					int							,					SCREAMING_SNAKE_CASE__							:					List[str]							,					SCREAMING_SNAKE_CASE__							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					str				):
          lowerCamelCase__           =		True
          lowerCamelCase__           =		TFRoFormerForCausalLM(config=SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }
          lowerCamelCase__           =		model(SCREAMING_SNAKE_CASE__				)['logits']
          self.parent.assertListEqual(
              list(prediction_scores.numpy().shape				)							,					[self.batch_size, self.seq_length, self.vocab_size]				)
    def      _UpperCamelCase       (   self							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					List[str]							,					SCREAMING_SNAKE_CASE__							:					str							,					SCREAMING_SNAKE_CASE__							:					Optional[Any]							,					SCREAMING_SNAKE_CASE__							:					Any							,					SCREAMING_SNAKE_CASE__							:					Optional[int]							,					SCREAMING_SNAKE_CASE__							:					Optional[int]							,					SCREAMING_SNAKE_CASE__							:					Tuple				):
          lowerCamelCase__           =		TFRoFormerForMaskedLM(config=SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }
          lowerCamelCase__           =		model(SCREAMING_SNAKE_CASE__				)
          self.parent.assertEqual(result.logits.shape							,					(self.batch_size, self.seq_length, self.vocab_size)				)
    def      _UpperCamelCase       (   self							:					Optional[Any]							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]							,					SCREAMING_SNAKE_CASE__							:					int							,					SCREAMING_SNAKE_CASE__							:					int							,					SCREAMING_SNAKE_CASE__							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					Any							,					SCREAMING_SNAKE_CASE__							:					List[Any]							,					SCREAMING_SNAKE_CASE__							:					Optional[Any]				):
          lowerCamelCase__           =		self.num_labels
          lowerCamelCase__           =		TFRoFormerForSequenceClassification(config=SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }

          lowerCamelCase__           =		model(SCREAMING_SNAKE_CASE__				)
          self.parent.assertEqual(result.logits.shape							,					(self.batch_size, self.num_labels)				)
    def      _UpperCamelCase       (   self							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]							,					SCREAMING_SNAKE_CASE__							:					List[str]							,					SCREAMING_SNAKE_CASE__							:					Any							,					SCREAMING_SNAKE_CASE__							:					Optional[int]							,					SCREAMING_SNAKE_CASE__							:					Any							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]							,					SCREAMING_SNAKE_CASE__							:					str				):
          lowerCamelCase__           =		self.num_choices
          lowerCamelCase__           =		TFRoFormerForMultipleChoice(config=SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE__							,					1				)							,					(1, self.num_choices, 1)				)
          lowerCamelCase__           =		tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE__							,					1				)							,					(1, self.num_choices, 1)				)
          lowerCamelCase__           =		tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE__							,					1				)							,					(1, self.num_choices, 1)				)
          lowerCamelCase__           =		{
              'input_ids': multiple_choice_inputs_ids,
              'attention_mask': multiple_choice_input_mask,
              'token_type_ids': multiple_choice_token_type_ids,
          }
          lowerCamelCase__           =		model(SCREAMING_SNAKE_CASE__				)
          self.parent.assertEqual(result.logits.shape							,					(self.batch_size, self.num_choices)				)
    def      _UpperCamelCase       (   self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					Optional[Any]							,					SCREAMING_SNAKE_CASE__							:					List[str]							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]							,					SCREAMING_SNAKE_CASE__							:					str							,					SCREAMING_SNAKE_CASE__							:					List[Any]							,					SCREAMING_SNAKE_CASE__							:					int							,					SCREAMING_SNAKE_CASE__							:					Dict				):
          lowerCamelCase__           =		self.num_labels
          lowerCamelCase__           =		TFRoFormerForTokenClassification(config=SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }
          lowerCamelCase__           =		model(SCREAMING_SNAKE_CASE__				)
          self.parent.assertEqual(result.logits.shape							,					(self.batch_size, self.seq_length, self.num_labels)				)
    def      _UpperCamelCase       (   self							:					int							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]							,					SCREAMING_SNAKE_CASE__							:					str							,					SCREAMING_SNAKE_CASE__							:					Any							,					SCREAMING_SNAKE_CASE__							:					Optional[int]							,					SCREAMING_SNAKE_CASE__							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					str							,					SCREAMING_SNAKE_CASE__							:					List[Any]				):
          lowerCamelCase__           =		TFRoFormerForQuestionAnswering(config=SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }

          lowerCamelCase__           =		model(SCREAMING_SNAKE_CASE__				)
          self.parent.assertEqual(result.start_logits.shape							,					(self.batch_size, self.seq_length)				)
          self.parent.assertEqual(result.end_logits.shape							,					(self.batch_size, self.seq_length)				)


    def      _UpperCamelCase       (   self							:					Union[str, Any]				):
          lowerCamelCase__           =		self.prepare_config_and_inputs()
          (
              (
              lowerCamelCase__  
          )    ,  (
              lowerCamelCase__  
          )    ,  (
              lowerCamelCase__  
          )    ,  (
              lowerCamelCase__  
          )    ,  (
              lowerCamelCase__  
          )    ,  (
              lowerCamelCase__  
          )    ,  (
              lowerCamelCase__  
          )    ,    
          )           =		config_and_inputs
          lowerCamelCase__           =		{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
          return config, inputs_dict





@require_tf
class   _a      (      SCREAMING_SNAKE_CASE_		,		SCREAMING_SNAKE_CASE_		,		unittest.TestCase ):
    a_    :				Tuple             =    (
        (
            TFRoFormerModel,
            TFRoFormerForCausalLM,
            TFRoFormerForMaskedLM,
            TFRoFormerForQuestionAnswering,
            TFRoFormerForSequenceClassification,
            TFRoFormerForTokenClassification,
            TFRoFormerForMultipleChoice,
        )
        if is_tf_available()
        else ()
    )
    a_    :				Union[str, Any]             =    (
        {
            'feature-extraction': TFRoFormerModel,
            'fill-mask': TFRoFormerForMaskedLM,
            'question-answering': TFRoFormerForQuestionAnswering,
            'text-classification': TFRoFormerForSequenceClassification,
            'text-generation': TFRoFormerForCausalLM,
            'token-classification': TFRoFormerForTokenClassification,
            'zero-shot': TFRoFormerForSequenceClassification,
        }
        if is_tf_available()
        else {}
    )

    a_    :				Dict             =    False
    a_    :				str             =    False
    def      _UpperCamelCase       (   self							:					Optional[int]							,					SCREAMING_SNAKE_CASE__							:					List[str]							,					SCREAMING_SNAKE_CASE__							:					Dict							,					SCREAMING_SNAKE_CASE__							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]							,					SCREAMING_SNAKE_CASE__							:					Tuple				):
          if pipeline_test_casse_name == "TextGenerationPipelineTests":
                return True

          return False
    def      _UpperCamelCase       (   self							:					Optional[int]				):
          lowerCamelCase__           =		TFRoFormerModelTester(self				)
          lowerCamelCase__           =		ConfigTester(self							,					config_class=SCREAMING_SNAKE_CASE__							,					hidden_size=37				)
    def      _UpperCamelCase       (   self							:					Any				):
          self.config_tester.run_common_tests()
    def      _UpperCamelCase       (   self							:					Union[str, Any]				):
          lowerCamelCase__           =		self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       (   self							:					Dict				):
          lowerCamelCase__           =		self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       (   self							:					Optional[Any]				):
          lowerCamelCase__           =		self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_lm_head(*SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       (   self							:					Dict				):
          lowerCamelCase__           =		self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       (   self							:					List[str]				):
          lowerCamelCase__           =		self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       (   self							:					str				):
          lowerCamelCase__           =		self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       (   self							:					Tuple				):
          lowerCamelCase__           =		self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE__				)


    @slow
    def      _UpperCamelCase       (   self							:					Union[str, Any]				):
          lowerCamelCase__           =		TFRoFormerModel.from_pretrained('junnyu/roformer_chinese_base'				)
          self.assertIsNotNone(SCREAMING_SNAKE_CASE__				)





@require_tf
class   _a      (      unittest.TestCase ):
    @slow
    def      _UpperCamelCase       (   self							:					int				):
          lowerCamelCase__           =		TFRoFormerForMaskedLM.from_pretrained('junnyu/roformer_chinese_base'				)
          lowerCamelCase__           =		tf.constant([[0, 1, 2, 3, 4, 5]]				)
          lowerCamelCase__           =		model(SCREAMING_SNAKE_CASE__				)[0]

          # TODO Replace vocab size
          lowerCamelCase__           =		5_00_00

          lowerCamelCase__           =		[1, 6, vocab_size]
          self.assertEqual(output.shape							,					SCREAMING_SNAKE_CASE__				)

          print(output[:, :3, :3]				)

          # TODO Replace values below with what was printed above.
          lowerCamelCase__           =		tf.constant(
              [
                  [
                      [-0.12_05_33_41, -1.0_26_49_01, 0.29_22_19_46],
                      [-1.5_13_37_83, 0.19_74_33, 0.15_19_06_07],
                      [-5.0_13_54_03, -3.90_02_56, -0.84_03_87_64],
                  ]
              ]				)
          tf.debugging.assert_near(output[:, :3, :3]							,					SCREAMING_SNAKE_CASE__							,					atol=1e-4				)





@require_tf
class   _a      (      unittest.TestCase ):
    a_    :				Optional[int]             =    1e-4
    def      _UpperCamelCase       (   self							:					str				):
          lowerCamelCase__           =		tf.constant([[4, 10]]				)
          lowerCamelCase__           =		TFRoFormerSinusoidalPositionalEmbedding(num_positions=6							,					embedding_dim=6				)

          lowerCamelCase__           =		emba(input_ids.shape				)
          lowerCamelCase__           =		tf.constant(
              [[0.00_00, 0.00_00, 0.00_00, 1.00_00, 1.00_00, 1.00_00], [0.84_15, 0.04_64, 0.00_22, 0.54_03, 0.99_89, 1.00_00]]				)

          tf.debugging.assert_near(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__							,					atol=self.tolerance				)


    def      _UpperCamelCase       (   self							:					str				):
          lowerCamelCase__           =		tf.constant(
              [
                  [0.00_00, 0.00_00, 0.00_00, 0.00_00, 0.00_00],
                  [0.84_15, 0.82_19, 0.80_20, 0.78_19, 0.76_17],
                  [0.90_93, 0.93_64, 0.95_81, 0.97_49, 0.98_70],
              ]				)
          lowerCamelCase__           =		TFRoFormerSinusoidalPositionalEmbedding(num_positions=5_12							,					embedding_dim=5_12				)
          emba([2, 16, 5_12]				)
          lowerCamelCase__           =		emba.weight[:3, :5]

          tf.debugging.assert_near(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__							,					atol=self.tolerance				)





@require_tf
class   _a      (      unittest.TestCase ):
    a_    :				Union[str, Any]             =    1e-4
    def      _UpperCamelCase       (   self							:					Any				):
          # 2,12,16,64
          lowerCamelCase__           =		tf.reshape(tf.range(2 * 12 * 16 * 64							,					dtype=tf.floataa				)							,					shape=(2, 12, 16, 64)				) / 1_00

          lowerCamelCase__           =		-tf.reshape(tf.range(2 * 12 * 16 * 64							,					dtype=tf.floataa				)							,					shape=(2, 12, 16, 64)				) / 1_00

          lowerCamelCase__           =		TFRoFormerSinusoidalPositionalEmbedding(num_positions=32							,					embedding_dim=64				)
          lowerCamelCase__           =		embed_positions([2, 16, 7_68]				)[None, None, :, :]

          lowerCamelCase__    ,  lowerCamelCase__           =		TFRoFormerSelfAttention.apply_rotary_position_embeddings(
              SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)

          lowerCamelCase__           =		tf.constant(
              [
                  [0.00_00, 0.01_00, 0.02_00, 0.03_00, 0.04_00, 0.05_00, 0.06_00, 0.07_00],
                  [-0.20_12, 0.88_97, 0.02_63, 0.94_01, 0.20_74, 0.94_63, 0.34_81, 0.93_43],
                  [-1.70_57, 0.62_71, -1.21_45, 1.38_97, -0.63_03, 1.76_47, -0.11_73, 1.89_85],
                  [-2.17_31, -1.63_97, -2.73_58, 0.28_54, -2.18_40, 1.71_83, -1.30_18, 2.48_71],
                  [0.27_17, -3.61_73, -2.92_06, -2.19_88, -3.66_38, 0.38_58, -2.91_55, 2.29_80],
                  [3.98_59, -2.15_80, -0.79_84, -4.49_04, -4.11_81, -2.02_52, -4.47_82, 1.12_53],
              ]				)
          lowerCamelCase__           =		tf.constant(
              [
                  [0.00_00, -0.01_00, -0.02_00, -0.03_00, -0.04_00, -0.05_00, -0.06_00, -0.07_00],
                  [0.20_12, -0.88_97, -0.02_63, -0.94_01, -0.20_74, -0.94_63, -0.34_81, -0.93_43],
                  [1.70_57, -0.62_71, 1.21_45, -1.38_97, 0.63_03, -1.76_47, 0.11_73, -1.89_85],
                  [2.17_31, 1.63_97, 2.73_58, -0.28_54, 2.18_40, -1.71_83, 1.30_18, -2.48_71],
                  [-0.27_17, 3.61_73, 2.92_06, 2.19_88, 3.66_38, -0.38_58, 2.91_55, -2.29_80],
                  [-3.98_59, 2.15_80, 0.79_84, 4.49_04, 4.11_81, 2.02_52, 4.47_82, -1.12_53],
              ]				)

          tf.debugging.assert_near(query_layer[0, 0, :6, :8]							,					SCREAMING_SNAKE_CASE__							,					atol=self.tolerance				)
          tf.debugging.assert_near(key_layer[0, 0, :6, :8]							,					SCREAMING_SNAKE_CASE__							,					atol=self.tolerance				)







"""simple docstring"""


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_nllb import NllbTokenizer
else:
   _snake_case          =							None


_snake_case          =							logging.get_logger(__name__)


_snake_case          =							{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

_snake_case          =							{
    "vocab_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"
        ),
    },
}

_snake_case          =							{
    "facebook/nllb-large-en-ro": 1024,
    "facebook/nllb-200-distilled-600M": 1024,
}

# fmt: off
_snake_case          =							["ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "ajp_Arab", "aka_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "cjk_Latn", "ckb_Arab", "crh_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ell_Grek", "eng_Latn", "epo_Latn", "est_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "pes_Arab", "fij_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "grn_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "knc_Arab", "knc_Latn", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kmb_Latn", "kon_Latn", "kor_Hang", "kmr_Latn", "lao_Laoo", "lvs_Latn", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "min_Latn", "mkd_Cyrl", "plt_Latn", "mlt_Latn", "mni_Beng", "khk_Cyrl", "mos_Latn", "mri_Latn", "zsm_Latn", "mya_Mymr", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "gaz_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pol_Latn", "por_Latn", "prs_Arab", "pbt_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Beng", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "som_Latn", "sot_Latn", "spa_Latn", "als_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tgl_Latn", "tha_Thai", "tir_Ethi", "taq_Latn", "taq_Tfng", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tzm_Tfng", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "vec_Latn", "vie_Latn", "war_Latn", "wol_Latn", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zho_Hans", "zho_Hant", "zul_Latn"]



class   _a      (      SCREAMING_SNAKE_CASE_ ):
    a_    :				Any             =    VOCAB_FILES_NAMES
    a_    :				Optional[Any]             =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a_    :				Optional[Any]             =    PRETRAINED_VOCAB_FILES_MAP
    a_    :				List[str]             =    ['input_ids', 'attention_mask']
    a_    :				Union[str, Any]             =    NllbTokenizer

    a_    :				List[int]             =    []
    a_    :				List[int]             =    []
    def __init__( self							:					int							,					SCREAMING_SNAKE_CASE__							:					List[Any]=None							,					SCREAMING_SNAKE_CASE__							:					Optional[int]=None							,					SCREAMING_SNAKE_CASE__							:					List[Any]="<s>"							,					SCREAMING_SNAKE_CASE__							:					List[str]="</s>"							,					SCREAMING_SNAKE_CASE__							:					Any="</s>"							,					SCREAMING_SNAKE_CASE__							:					List[str]="<s>"							,					SCREAMING_SNAKE_CASE__							:					Tuple="<unk>"							,					SCREAMING_SNAKE_CASE__							:					Optional[int]="<pad>"							,					SCREAMING_SNAKE_CASE__							:					Any="<mask>"							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]=None							,					SCREAMING_SNAKE_CASE__							:					int=None							,					SCREAMING_SNAKE_CASE__							:					Dict=None							,					SCREAMING_SNAKE_CASE__							:					Tuple=False							,					**SCREAMING_SNAKE_CASE__							:					str							,					):
          # Mask token behave like a normal word, i.e. include the space before it
          lowerCamelCase__           =		AddedToken(SCREAMING_SNAKE_CASE__							,					lstrip=SCREAMING_SNAKE_CASE__							,					rstrip=SCREAMING_SNAKE_CASE__				) if isinstance(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				) else mask_token
          lowerCamelCase__           =		legacy_behaviour
          super().__init__(
              vocab_file=SCREAMING_SNAKE_CASE__							,					tokenizer_file=SCREAMING_SNAKE_CASE__							,					bos_token=SCREAMING_SNAKE_CASE__							,					eos_token=SCREAMING_SNAKE_CASE__							,					sep_token=SCREAMING_SNAKE_CASE__							,					cls_token=SCREAMING_SNAKE_CASE__							,					unk_token=SCREAMING_SNAKE_CASE__							,					pad_token=SCREAMING_SNAKE_CASE__							,					mask_token=SCREAMING_SNAKE_CASE__							,					src_lang=SCREAMING_SNAKE_CASE__							,					tgt_lang=SCREAMING_SNAKE_CASE__							,					additional_special_tokens=SCREAMING_SNAKE_CASE__							,					legacy_behaviour=SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__							,					)

          lowerCamelCase__           =		vocab_file
          lowerCamelCase__           =		False if not self.vocab_file else True

          lowerCamelCase__           =		FAIRSEQ_LANGUAGE_CODES.copy()

          if additional_special_tokens is not None:
                # Only add those special tokens if they are not already there.
                _additional_special_tokens.extend(
                    [t for t in additional_special_tokens if t not in _additional_special_tokens]				)

          self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}				)
          lowerCamelCase__           =		{
              lang_code: self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__				) for lang_code in FAIRSEQ_LANGUAGE_CODES
          }

          lowerCamelCase__           =		src_lang if src_lang is not None else 'eng_Latn'
          lowerCamelCase__           =		self.convert_tokens_to_ids(self._src_lang				)
          lowerCamelCase__           =		tgt_lang
          self.set_src_lang_special_tokens(self._src_lang				)
    @property
    def      _UpperCamelCase       ( self							:					str				):
          return self._src_lang
    @src_lang.setter
    def      _UpperCamelCase       ( self							:					int							,					SCREAMING_SNAKE_CASE__							:					str				):
          lowerCamelCase__           =		new_src_lang
          self.set_src_lang_special_tokens(self._src_lang				)
    def      _UpperCamelCase       ( self							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					List[int]							,					SCREAMING_SNAKE_CASE__							:					Optional[List[int]] = None				):
          if token_ids_a is None:
                return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
    def      _UpperCamelCase       ( self							:					Optional[Any]							,					SCREAMING_SNAKE_CASE__							:					List[int]							,					SCREAMING_SNAKE_CASE__							:					Optional[List[int]] = None				):
          lowerCamelCase__           =		[self.sep_token_id]
          lowerCamelCase__           =		[self.cls_token_id]

          if token_ids_a is None:
                return len(cls + token_ids_a + sep				) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]
    def      _UpperCamelCase       ( self							:					str							,					SCREAMING_SNAKE_CASE__							:					str							,					SCREAMING_SNAKE_CASE__							:					str							,					SCREAMING_SNAKE_CASE__							:					Optional[str]							,					SCREAMING_SNAKE_CASE__							:					Optional[str]							,					**SCREAMING_SNAKE_CASE__							:					Optional[int]				):
          if src_lang is None or tgt_lang is None:
                raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'				)
          lowerCamelCase__           =		src_lang
          lowerCamelCase__           =		self(SCREAMING_SNAKE_CASE__							,					add_special_tokens=SCREAMING_SNAKE_CASE__							,					return_tensors=SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		tgt_lang_id
          return inputs
    def      _UpperCamelCase       ( self							:					str							,					SCREAMING_SNAKE_CASE__							:					List[str]							,					SCREAMING_SNAKE_CASE__							:					str = "eng_Latn"							,					SCREAMING_SNAKE_CASE__							:					Optional[List[str]] = None							,					SCREAMING_SNAKE_CASE__							:					str = "fra_Latn"							,					**SCREAMING_SNAKE_CASE__							:					Dict							,					):
          lowerCamelCase__           =		src_lang
          lowerCamelCase__           =		tgt_lang
          return super().prepare_seqaseq_batch(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       ( self							:					List[str]				):
          return self.set_src_lang_special_tokens(self.src_lang				)
    def      _UpperCamelCase       ( self							:					List[Any]				):
          return self.set_tgt_lang_special_tokens(self.tgt_lang				)
    def      _UpperCamelCase       ( self							:					str							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]				):
          lowerCamelCase__           =		self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__				)

          if self.legacy_behaviour:
                lowerCamelCase__           =		[]
                lowerCamelCase__           =		[self.eos_token_id, self.cur_lang_code]
          else:
                lowerCamelCase__           =		[self.cur_lang_code]
                lowerCamelCase__           =		[self.eos_token_id]

          lowerCamelCase__           =		self.convert_ids_to_tokens(self.prefix_tokens				)
          lowerCamelCase__           =		self.convert_ids_to_tokens(self.suffix_tokens				)

          lowerCamelCase__           =		processors.TemplateProcessing(
              single=prefix_tokens_str + ['$A'] + suffix_tokens_str							,					pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str							,					special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,					self.prefix_tokens + self.suffix_tokens				)				)							,					)
    def      _UpperCamelCase       ( self							:					Union[str, Any]							,					SCREAMING_SNAKE_CASE__							:					str				):
          lowerCamelCase__           =		self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__				)
          if self.legacy_behaviour:
                lowerCamelCase__           =		[]
                lowerCamelCase__           =		[self.eos_token_id, self.cur_lang_code]
          else:
                lowerCamelCase__           =		[self.cur_lang_code]
                lowerCamelCase__           =		[self.eos_token_id]

          lowerCamelCase__           =		self.convert_ids_to_tokens(self.prefix_tokens				)
          lowerCamelCase__           =		self.convert_ids_to_tokens(self.suffix_tokens				)

          lowerCamelCase__           =		processors.TemplateProcessing(
              single=prefix_tokens_str + ['$A'] + suffix_tokens_str							,					pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str							,					special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,					self.prefix_tokens + self.suffix_tokens				)				)							,					)


    def      _UpperCamelCase       ( self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					str							,					SCREAMING_SNAKE_CASE__							:					Optional[str] = None				):
          if not self.can_save_slow_tokenizer:
                raise ValueError(
                    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                    'tokenizer.'				)

          if not os.path.isdir(SCREAMING_SNAKE_CASE__				):
                logger.error(F'Vocabulary path ({save_directory}) should be a directory.'				)
                return
          lowerCamelCase__           =		os.path.join(
              SCREAMING_SNAKE_CASE__							,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)

          if os.path.abspath(self.vocab_file				) != os.path.abspath(SCREAMING_SNAKE_CASE__				):
                copyfile(self.vocab_file							,					SCREAMING_SNAKE_CASE__				)

          return (out_vocab_file,)



