



"""simple docstring"""






import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__snake_case							:     Optional[int]   =					logging.get_logger(__name__)

__snake_case							:     List[Any]   =					'▁'

__snake_case							:     int   =					{
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

__snake_case							:     int   =					{
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

__snake_case							:     Any   =					{
    'facebook/m2m100_418M': 1_024,
}

# fmt: off
__snake_case							:     List[Any]   =					{
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}



class     A__     (						__SCREAMING_SNAKE_CASE							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE					   =			VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE					   =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE					   =			PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE					   =			['input_ids', 'attention_mask']

	SCREAMING_SNAKE_CASE					   =			[]
	SCREAMING_SNAKE_CASE					   =			[]

	def __init__(  self:	List[str] ,    _SCREAMING_SNAKE_CASE:	List[Any] ,    _SCREAMING_SNAKE_CASE:	Dict ,    _SCREAMING_SNAKE_CASE:	Optional[int]=None ,    _SCREAMING_SNAKE_CASE:	str=None ,    _SCREAMING_SNAKE_CASE:	Any="<s>" ,    _SCREAMING_SNAKE_CASE:	int="</s>" ,    _SCREAMING_SNAKE_CASE:	int="</s>" ,    _SCREAMING_SNAKE_CASE:	List[str]="<pad>" ,    _SCREAMING_SNAKE_CASE:	Tuple="<unk>" ,    _SCREAMING_SNAKE_CASE:	List[str]="m2m100" ,    _SCREAMING_SNAKE_CASE:	Optional[Dict[str, Any]] = None ,    _SCREAMING_SNAKE_CASE:	List[str]=8 ,    **_SCREAMING_SNAKE_CASE:	List[Any] ,    )       -> None:

				"""simple docstring"""

				__lowerCAmelCase   :						Dict				= {} if sp_model_kwargs is None else sp_model_kwargs

				__lowerCAmelCase   :						Any				= language_codes
				__lowerCAmelCase   :						str				= FAIRSEQ_LANGUAGE_CODES[language_codes]
				__lowerCAmelCase   :						Optional[Any]				= {lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

				__lowerCAmelCase   :						Tuple				= kwargs.get("additional_special_tokens" ,    [])
				kwargs["additional_special_tokens"] += [
				    self.get_lang_token(_SCREAMING_SNAKE_CASE)
				    for lang_code in fairseq_language_code
				    if self.get_lang_token(_SCREAMING_SNAKE_CASE) not in kwargs["additional_special_tokens"]
				]

				super().__init__(
				    src_lang=_SCREAMING_SNAKE_CASE ,    tgt_lang=_SCREAMING_SNAKE_CASE ,    bos_token=_SCREAMING_SNAKE_CASE ,    eos_token=_SCREAMING_SNAKE_CASE ,    sep_token=_SCREAMING_SNAKE_CASE ,    unk_token=_SCREAMING_SNAKE_CASE ,    pad_token=_SCREAMING_SNAKE_CASE ,    language_codes=_SCREAMING_SNAKE_CASE ,    sp_model_kwargs=self.sp_model_kwargs ,    num_madeup_words=_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE ,    )

				__lowerCAmelCase   :						Optional[Any]				= vocab_file
				__lowerCAmelCase   :						List[Any]				= load_json(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						Tuple				= {v: k for k, v in self.encoder.items()}
				__lowerCAmelCase   :						int				= spm_file
				__lowerCAmelCase   :						Tuple				= load_spm(_SCREAMING_SNAKE_CASE ,    self.sp_model_kwargs)

				__lowerCAmelCase   :						Any				= len(self.encoder)

				__lowerCAmelCase   :						Dict				= {
				    self.get_lang_token(_SCREAMING_SNAKE_CASE): self.encoder_size + i for i, lang_code in enumerate(_SCREAMING_SNAKE_CASE)
				}
				__lowerCAmelCase   :						Dict				= {lang_code: self.encoder_size + i for i, lang_code in enumerate(_SCREAMING_SNAKE_CASE)}
				__lowerCAmelCase   :						List[Any]				= {v: k for k, v in self.lang_token_to_id.items()}

				__lowerCAmelCase   :						List[str]				= src_lang if src_lang is not None else "en"
				__lowerCAmelCase   :						str				= tgt_lang
				__lowerCAmelCase   :						Dict				= self.get_lang_id(self._src_lang)
				self.set_src_lang_special_tokens(self._src_lang)

				__lowerCAmelCase   :						str				= num_madeup_words

	@property
	def 							_SCREAMING_SNAKE_CASE     (  self:	str)       -> int:

				"""simple docstring"""

				return len(self.encoder) + len(self.lang_token_to_id)

	@property
	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict)       -> str:

				"""simple docstring"""

				return self._src_lang

	@src_lang.setter
	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[int] ,    _SCREAMING_SNAKE_CASE:	str)       -> None:

				"""simple docstring"""

				__lowerCAmelCase   :						List[Any]				= new_src_lang
				self.set_src_lang_special_tokens(self._src_lang)

	def 							_SCREAMING_SNAKE_CASE     (  self:	int ,    _SCREAMING_SNAKE_CASE:	str)       -> List[str]:

				"""simple docstring"""

				return self.sp_model.encode(_SCREAMING_SNAKE_CASE ,    out_type=_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[int] ,    _SCREAMING_SNAKE_CASE:	int)       -> Optional[Any]:

				"""simple docstring"""

				if token in self.lang_token_to_id:
							return self.lang_token_to_id[token]
				return self.encoder.get(_SCREAMING_SNAKE_CASE ,    self.encoder[self.unk_token])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[int] ,    _SCREAMING_SNAKE_CASE:	int)       -> str:

				"""simple docstring"""

				if index in self.id_to_lang_token:
							return self.id_to_lang_token[index]
				return self.decoder.get(_SCREAMING_SNAKE_CASE ,    self.unk_token)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict ,    _SCREAMING_SNAKE_CASE:	str)       -> List[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						Union[str, Any]				= []
				__lowerCAmelCase   :						List[str]				= ""
				for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
										out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE) + token
										__lowerCAmelCase   :						Dict				= []
							else:
										current_sub_tokens.append(_SCREAMING_SNAKE_CASE)
				out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE)
				return out_string.strip()

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str] ,    _SCREAMING_SNAKE_CASE:	List[int] ,    _SCREAMING_SNAKE_CASE:	Optional[List[int]] = None ,    _SCREAMING_SNAKE_CASE:	bool = False)       -> List[int]:

				"""simple docstring"""

				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=_SCREAMING_SNAKE_CASE ,    token_ids_a=_SCREAMING_SNAKE_CASE ,    already_has_special_tokens=_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase   :						Optional[Any]				= [1] * len(self.prefix_tokens)
				__lowerCAmelCase   :						List[str]				= [1] * len(self.suffix_tokens)
				if token_ids_a is None:
							return prefix_ones + ([0] * len(_SCREAMING_SNAKE_CASE)) + suffix_ones
				return prefix_ones + ([0] * len(_SCREAMING_SNAKE_CASE)) + ([0] * len(_SCREAMING_SNAKE_CASE)) + suffix_ones

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str] ,    _SCREAMING_SNAKE_CASE:	List[int] ,    _SCREAMING_SNAKE_CASE:	Optional[List[int]] = None)       -> List[int]:

				"""simple docstring"""

				if token_ids_a is None:
							return self.prefix_tokens + token_ids_a + self.suffix_tokens
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any])       -> Dict:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE): i for i in range(self.vocab_size)}
				vocab.update(self.added_tokens_encoder)
				return vocab

	def __getstate__(  self:	List[str])       -> Dict:

				"""simple docstring"""

				__lowerCAmelCase   :						List[str]				= self.__dict__.copy()
				__lowerCAmelCase   :						Optional[int]				= None
				return state

	def __setstate__(  self:	Optional[Any] ,    _SCREAMING_SNAKE_CASE:	Dict)       -> None:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= d

				# for backward compatibility
				if not hasattr(self ,    "sp_model_kwargs"):
							__lowerCAmelCase   :						int				= {}

				__lowerCAmelCase   :						Optional[int]				= load_spm(self.spm_file ,    self.sp_model_kwargs)

	def 							_SCREAMING_SNAKE_CASE     (  self:	int ,    _SCREAMING_SNAKE_CASE:	str ,    _SCREAMING_SNAKE_CASE:	Optional[str] = None)       -> Tuple[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						List[Any]				= Path(_SCREAMING_SNAKE_CASE)
				if not save_dir.is_dir():
							raise OSError(F"""{save_directory} should be a directory""")
				__lowerCAmelCase   :						Union[str, Any]				= save_dir / (
				    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
				)
				__lowerCAmelCase   :						Dict				= save_dir / (
				    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
				)

				save_json(self.encoder ,    _SCREAMING_SNAKE_CASE)

				if os.path.abspath(self.spm_file) != os.path.abspath(_SCREAMING_SNAKE_CASE) and os.path.isfile(self.spm_file):
							copyfile(self.spm_file ,    _SCREAMING_SNAKE_CASE)
				elif not os.path.isfile(self.spm_file):
							with open(_SCREAMING_SNAKE_CASE ,    "wb") as fi:
										__lowerCAmelCase   :						List[str]				= self.sp_model.serialized_model_proto()
										fi.write(_SCREAMING_SNAKE_CASE)

				return (str(_SCREAMING_SNAKE_CASE), str(_SCREAMING_SNAKE_CASE))

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[int] ,    _SCREAMING_SNAKE_CASE:	List[str] ,    _SCREAMING_SNAKE_CASE:	str = "en" ,    _SCREAMING_SNAKE_CASE:	Optional[List[str]] = None ,    _SCREAMING_SNAKE_CASE:	str = "ro" ,    **_SCREAMING_SNAKE_CASE:	int ,    )       -> BatchEncoding:

				"""simple docstring"""

				__lowerCAmelCase   :						Union[str, Any]				= src_lang
				__lowerCAmelCase   :						Dict				= tgt_lang
				self.set_src_lang_special_tokens(self.src_lang)
				return super().prepare_seqaseq_batch(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any ,    _SCREAMING_SNAKE_CASE:	Union[str, Any] ,    _SCREAMING_SNAKE_CASE:	Optional[str] ,    _SCREAMING_SNAKE_CASE:	Optional[str] ,    **_SCREAMING_SNAKE_CASE:	Tuple)       -> Any:

				"""simple docstring"""

				if src_lang is None or tgt_lang is None:
							raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model")
				__lowerCAmelCase   :						List[str]				= src_lang
				__lowerCAmelCase   :						Dict				= self(_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						List[Any]				= self.get_lang_id(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						List[Any]				= tgt_lang_id
				return inputs

	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any])       -> Tuple:

				"""simple docstring"""

				self.set_src_lang_special_tokens(self.src_lang)

	def 							_SCREAMING_SNAKE_CASE     (  self:	int)       -> Union[str, Any]:

				"""simple docstring"""

				self.set_tgt_lang_special_tokens(self.tgt_lang)

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any] ,    _SCREAMING_SNAKE_CASE:	str)       -> None:

				"""simple docstring"""

				__lowerCAmelCase   :						int				= self.get_lang_token(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						List[str]				= self.lang_token_to_id[lang_token]
				__lowerCAmelCase   :						Optional[Any]				= [self.cur_lang_id]
				__lowerCAmelCase   :						int				= [self.eos_token_id]

	def 							_SCREAMING_SNAKE_CASE     (  self:	int ,    _SCREAMING_SNAKE_CASE:	str)       -> None:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= self.get_lang_token(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						Tuple				= self.lang_token_to_id[lang_token]
				__lowerCAmelCase   :						Union[str, Any]				= [self.cur_lang_id]
				__lowerCAmelCase   :						Union[str, Any]				= [self.eos_token_id]

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any] ,    _SCREAMING_SNAKE_CASE:	str)       -> str:

				"""simple docstring"""

				return self.lang_code_to_token[lang]







	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict ,    _SCREAMING_SNAKE_CASE:	str)       -> int:

				"""simple docstring"""

				__lowerCAmelCase   :						Optional[Any]				= self.get_lang_token(_SCREAMING_SNAKE_CASE)
				return self.lang_token_to_id[lang_token]






def     _lowercase       (						__snake_case      ,__snake_case			)		->     sentencepiece.SentencePieceProcessor:
			__lowerCAmelCase   :						Optional[int]				= sentencepiece.SentencePieceProcessor(**__snake_case			)
			spm.Load(str(__snake_case			)			)
			return spm


def     _lowercase       (						__snake_case			)		->     Union[Dict, List]:
			with open(__snake_case      ,"r"			) as f:
						return json.load(__snake_case			)





def     _lowercase       (						__snake_case      ,__snake_case			)		->     None:
			with open(__snake_case      ,"w"			) as f:
						json.dump(__snake_case      ,__snake_case      ,indent=2			)




"""simple docstring"""






import math


def     _lowercase       (						__snake_case      ,__snake_case			)		->     float:
			return math.pow(__snake_case      ,2			) - a


def     _lowercase       (						__snake_case			)		->     float:
			return 2 * x


def     _lowercase       (						__snake_case			)		->     float:
			__lowerCAmelCase   :						List[str]				= 2.0

			while start <= a:
						__lowerCAmelCase   :						Union[str, Any]				= math.pow(__snake_case      ,2			)

			return start


def     _lowercase       (						__snake_case      ,__snake_case = 9_999      ,__snake_case = 0.00000000000001			)		->     float:

			if a < 0:
						raise ValueError("math domain error"			)

			__lowerCAmelCase   :						List[Any]				= get_initial_point(__snake_case			)

			for _ in range(__snake_case			):
						__lowerCAmelCase   :						Union[str, Any]				= value
						__lowerCAmelCase   :						List[str]				= value - fx(__snake_case      ,__snake_case			) / fx_derivative(__snake_case			)
						if abs(prev_value - value			) < tolerance:
									return value

			return value


if __name__ == "__main__":
	from doctest import testmod

	testmod()