



import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


snake_case__				= logging.get_logger(__name__)

snake_case__				= '▁'

snake_case__				= {
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

snake_case__				= {
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

snake_case__				= {
    'facebook/m2m100_418M': 1024,
}

# fmt: off
snake_case__				= {
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}



class  UpperCamelCase				( __lowercase ):

     '''simple docstring'''

     A_										=  VOCAB_FILES_NAMES
     A_										=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     A_										=  PRETRAINED_VOCAB_FILES_MAP
     A_										=  ['input_ids', 'attention_mask']

     A_										=  []
     A_										=  []



     def __init__(					self				,				A_				,				A_				,				A_=None				,				A_=None				,				A_="<s>"				,				A_="</s>"				,				A_="</s>"				,				A_="<pad>"				,				A_="<unk>"				,				A_="m2m100"				,				A_ = None				,				A_=8				,				**A_				,				) ->				None:






           """simple docstring"""
           _lowerCamelCase			    =				{} if sp_model_kwargs is None else sp_model_kwargs

           _lowerCamelCase			    =				language_codes
           _lowerCamelCase			    =				FAIRSEQ_LANGUAGE_CODES[language_codes]
           _lowerCamelCase			    =				{lang_code: F'__{lang_code}__' for lang_code in fairseq_language_code}

           _lowerCamelCase			    =				kwargs.get('''additional_special_tokens'''				,				[]						)
           kwargs["additional_special_tokens"] += [
               self.get_lang_token(A_						)
               for lang_code in fairseq_language_code
               if self.get_lang_token(A_						) not in kwargs["additional_special_tokens"]
           ]

           super().__init__(
               src_lang=A_				,				tgt_lang=A_				,				bos_token=A_				,				eos_token=A_				,				sep_token=A_				,				unk_token=A_				,				pad_token=A_				,				language_codes=A_				,				sp_model_kwargs=self.sp_model_kwargs				,				num_madeup_words=A_				,				**A_				,				)

           _lowerCamelCase			    =				vocab_file
           _lowerCamelCase			    =				load_json(A_						)
           _lowerCamelCase			    =				{v: k for k, v in self.encoder.items()}
           _lowerCamelCase			    =				spm_file
           _lowerCamelCase			    =				load_spm(A_				,				self.sp_model_kwargs						)

           _lowerCamelCase			    =				len(self.encoder						)

           _lowerCamelCase			    =				{
               self.get_lang_token(A_						): self.encoder_size + i for i, lang_code in enumerate(A_						)
           }
           _lowerCamelCase			    =				{lang_code: self.encoder_size + i for i, lang_code in enumerate(A_						)}
           _lowerCamelCase			    =				{v: k for k, v in self.lang_token_to_id.items()}

           _lowerCamelCase			    =				src_lang if src_lang is not None else '''en'''
           _lowerCamelCase			    =				tgt_lang
           _lowerCamelCase			    =				self.get_lang_id(self._src_lang						)
           self.set_src_lang_special_tokens(self._src_lang						)

           _lowerCamelCase			    =				num_madeup_words



     @property
     def 		UpperCamelCase_						(					self						) ->				int:






           """simple docstring"""
           return len(self.encoder						) + len(self.lang_token_to_id						)



     @property
     def 		UpperCamelCase_						(					self						) ->				str:






           """simple docstring"""
           return self._src_lang



     @src_lang.setter
     def 		UpperCamelCase_						(					self				,				A_						) ->				None:






           """simple docstring"""
           _lowerCamelCase			    =				new_src_lang
           self.set_src_lang_special_tokens(self._src_lang						)



     def 		UpperCamelCase_						(					self				,				A_						) ->				List[str]:






           """simple docstring"""
           return self.sp_model.encode(A_				,				out_type=A_						)



     def 		UpperCamelCase_						(					self				,				A_						) ->				List[Any]:






           """simple docstring"""
           if token in self.lang_token_to_id:
                 return self.lang_token_to_id[token]
           return self.encoder.get(A_				,				self.encoder[self.unk_token]						)



     def 		UpperCamelCase_						(					self				,				A_						) ->				str:






           """simple docstring"""
           if index in self.id_to_lang_token:
                 return self.id_to_lang_token[index]
           return self.decoder.get(A_				,				self.unk_token						)



     def 		UpperCamelCase_						(					self				,				A_						) ->				Optional[int]:






           """simple docstring"""
           _lowerCamelCase			    =				[]
           _lowerCamelCase			    =				''''''
           for token in tokens:
                 # make sure that special tokens are not decoded using sentencepiece model
                 if token in self.all_special_tokens:
                       out_string += self.sp_model.decode(A_						) + token
                       _lowerCamelCase			    =				[]
                 else:
                       current_sub_tokens.append(A_						)
           out_string += self.sp_model.decode(A_						)
           return out_string.strip()



     def 		UpperCamelCase_						(					self				,				A_				,				A_ = None				,				A_ = False						) ->				List[int]:






           """simple docstring"""

           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=A_				,				token_ids_a=A_				,				already_has_special_tokens=A_						)

           _lowerCamelCase			    =				[1] * len(self.prefix_tokens						)
           _lowerCamelCase			    =				[1] * len(self.suffix_tokens						)
           if token_ids_a is None:
                 return prefix_ones + ([0] * len(A_						)) + suffix_ones
           return prefix_ones + ([0] * len(A_						)) + ([0] * len(A_						)) + suffix_ones



     def 		UpperCamelCase_						(					self				,				A_				,				A_ = None						) ->				List[int]:






           """simple docstring"""
           if token_ids_a is None:
                 return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



     def 		UpperCamelCase_						(					self						) ->				Dict:






           """simple docstring"""
           _lowerCamelCase			    =				{self.convert_ids_to_tokens(A_						): i for i in range(self.vocab_size						)}
           vocab.update(self.added_tokens_encoder						)
           return vocab



     def __getstate__(					self						) ->				Dict:






           """simple docstring"""
           _lowerCamelCase			    =				self.__dict__.copy()
           _lowerCamelCase			    =				None
           return state



     def __setstate__(					self				,				A_						) ->				None:






           """simple docstring"""
           _lowerCamelCase			    =				d

           # for backward compatibility
           if not hasattr(self				,				'''sp_model_kwargs'''						):
                 _lowerCamelCase			    =				{}

           _lowerCamelCase			    =				load_spm(self.spm_file				,				self.sp_model_kwargs						)



     def 		UpperCamelCase_						(					self				,				A_				,				A_ = None						) ->				Tuple[str]:






           """simple docstring"""
           _lowerCamelCase			    =				Path(A_						)
           if not save_dir.is_dir():
                 raise OSError(F'{save_directory} should be a directory'						)
           _lowerCamelCase			    =				save_dir / (
               (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
           )
           _lowerCamelCase			    =				save_dir / (
               (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
           )

           save_json(self.encoder				,				A_						)

           if os.path.abspath(self.spm_file						) != os.path.abspath(A_						) and os.path.isfile(self.spm_file						):
                 copyfile(self.spm_file				,				A_						)
           elif not os.path.isfile(self.spm_file						):
                 with open(A_				,				'''wb'''						) as fi:
                       _lowerCamelCase			    =				self.sp_model.serialized_model_proto()
                       fi.write(A_						)

           return (str(A_						), str(A_						))



     def 		UpperCamelCase_						(					self				,				A_				,				A_ = "en"				,				A_ = None				,				A_ = "ro"				,				**A_				,				) ->				BatchEncoding:






           """simple docstring"""
           _lowerCamelCase			    =				src_lang
           _lowerCamelCase			    =				tgt_lang
           self.set_src_lang_special_tokens(self.src_lang						)
           return super().prepare_seqaseq_batch(A_				,				A_				,				**A_						)



     def 		UpperCamelCase_						(					self				,				A_				,				A_				,				A_				,				**A_						) ->				Tuple:






           """simple docstring"""
           if src_lang is None or tgt_lang is None:
                 raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''						)
           _lowerCamelCase			    =				src_lang
           _lowerCamelCase			    =				self(A_				,				add_special_tokens=A_				,				**A_						)
           _lowerCamelCase			    =				self.get_lang_id(A_						)
           _lowerCamelCase			    =				tgt_lang_id
           return inputs



     def 		UpperCamelCase_						(					self						) ->				Optional[int]:






           """simple docstring"""
           self.set_src_lang_special_tokens(self.src_lang						)



     def 		UpperCamelCase_						(					self						) ->				Any:






           """simple docstring"""
           self.set_tgt_lang_special_tokens(self.tgt_lang						)



     def 		UpperCamelCase_						(					self				,				A_						) ->				None:






           """simple docstring"""
           _lowerCamelCase			    =				self.get_lang_token(A_						)
           _lowerCamelCase			    =				self.lang_token_to_id[lang_token]
           _lowerCamelCase			    =				[self.cur_lang_id]
           _lowerCamelCase			    =				[self.eos_token_id]



     def 		UpperCamelCase_						(					self				,				A_						) ->				None:






           """simple docstring"""
           _lowerCamelCase			    =				self.get_lang_token(A_						)
           _lowerCamelCase			    =				self.lang_token_to_id[lang_token]
           _lowerCamelCase			    =				[self.cur_lang_id]
           _lowerCamelCase			    =				[self.eos_token_id]



     def 		UpperCamelCase_						(					self				,				A_						) ->				str:






           """simple docstring"""
           return self.lang_code_to_token[lang]





     def 		UpperCamelCase_						(					self				,				A_						) ->				int:






           """simple docstring"""
           _lowerCamelCase			    =				self.get_lang_token(A_						)
           return self.lang_token_to_id[lang_token]





def       __magic_name__(  __UpperCAmelCase ,						__UpperCAmelCase		)						->    sentencepiece.SentencePieceProcessor:



      '''simple docstring'''

      _lowerCamelCase			    =				sentencepiece.SentencePieceProcessor(**__UpperCAmelCase		)
      spm.Load(str(__UpperCAmelCase		)		)
      return spm
def       __magic_name__(  __UpperCAmelCase		)						->    Union[Dict, List]:



      '''simple docstring'''

      with open(__UpperCAmelCase ,						'''r'''		) as f:
            return json.load(__UpperCAmelCase		)




def       __magic_name__(  __UpperCAmelCase ,						__UpperCAmelCase		)						->    None:



      '''simple docstring'''

      with open(__UpperCAmelCase ,						'''w'''		) as f:
            json.dump(__UpperCAmelCase ,						__UpperCAmelCase ,						indent=2		)
from .data_collator import (
    DataCollatorForLanguageModeling,
    DataCollatorForPermutationLanguageModeling,
    DataCollatorForSeqaSeq,
    DataCollatorForSOP,
    DataCollatorForTokenClassification,
    DataCollatorForWholeWordMask,
    DataCollatorWithPadding,
    DefaultDataCollator,
    default_data_collator,
)
from .metrics import glue_compute_metrics, xnli_compute_metrics
from .processors import (
    DataProcessor,
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
    SquadExample,
    SquadFeatures,
    SquadVaProcessor,
    SquadVaProcessor,
    glue_convert_examples_to_features,
    glue_output_modes,
    glue_processors,
    glue_tasks_num_labels,
    squad_convert_examples_to_features,
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)