



import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__ =						logging.get_logger(__name__)

UpperCamelCase__ =						"▁"

UpperCamelCase__ =						{
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

UpperCamelCase__ =						{
    "vocab_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json",
    },
    "spm_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json",
    },
}

UpperCamelCase__ =						{
    "facebook/m2m100_418M": 10_24,
}

# fmt: off
UpperCamelCase__ =						{
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"],
    "wmt21": ["en", "ha", "is", "ja", "cs", "ru", "zh", "de"]
}
class 			_lowerCAmelCase   ( _A			):






			"""simple docstring"""





			lowercase__      :  List[str]      =			VOCAB_FILES_NAMES
			lowercase__      :  Optional[int]      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowercase__      :  int      =			PRETRAINED_VOCAB_FILES_MAP
			lowercase__      :  str      =			["""input_ids""", """attention_mask"""]

			lowercase__      :  Optional[Any]      =			[]
			lowercase__      :  Union[str, Any]      =			[]





			def __init__(				self	:       Dict			,     lowercase	:       Optional[Any]			,     lowercase	:       Optional[int]			,     lowercase	:       Tuple=None			,     lowercase	:       int=None			,     lowercase	:       Union[str, Any]="<s>"			,     lowercase	:       int="</s>"			,     lowercase	:       str="</s>"			,     lowercase	:       int="<pad>"			,     lowercase	:       List[str]="<unk>"			,     lowercase	:       str="m2m100"			,     lowercase	:       Optional[Dict[str, Any]] = None			,     lowercase	:       int=8			,     **lowercase	:       List[Any]			,     )						->					None:
									"""simple docstring"""







									__lowercase					=				{} if sp_model_kwargs is None else sp_model_kwargs

									__lowercase					=				language_codes
									__lowercase					=				FAIRSEQ_LANGUAGE_CODES[language_codes]
									__lowercase					=				{lang_code: F"__{lang_code}__" for lang_code in fairseq_language_code}

									__lowercase					=				kwargs.get("""additional_special_tokens"""			,     []   )
									kwargs["additional_special_tokens"] += [
									    self.get_lang_token(__lowerCamelCase   )
									    for lang_code in fairseq_language_code
									    if self.get_lang_token(__lowerCamelCase   ) not in kwargs["additional_special_tokens"]
									]

									super().__init__(
									    src_lang=__lowerCamelCase			,     tgt_lang=__lowerCamelCase			,     bos_token=__lowerCamelCase			,     eos_token=__lowerCamelCase			,     sep_token=__lowerCamelCase			,     unk_token=__lowerCamelCase			,     pad_token=__lowerCamelCase			,     language_codes=__lowerCamelCase			,     sp_model_kwargs=self.sp_model_kwargs			,     num_madeup_words=__lowerCamelCase			,     **__lowerCamelCase			,     )

									__lowercase					=				vocab_file
									__lowercase					=				load_json(__lowerCamelCase   )
									__lowercase					=				{v: k for k, v in self.encoder.items()}
									__lowercase					=				spm_file
									__lowercase					=				load_spm(__lowerCamelCase			,     self.sp_model_kwargs   )

									__lowercase					=				len(self.encoder   )

									__lowercase					=				{
									    self.get_lang_token(__lowerCamelCase   ): self.encoder_size + i for i, lang_code in enumerate(__lowerCamelCase   )
									}
									__lowercase					=				{lang_code: self.encoder_size + i for i, lang_code in enumerate(__lowerCamelCase   )}
									__lowercase					=				{v: k for k, v in self.lang_token_to_id.items()}

									__lowercase					=				src_lang if src_lang is not None else "en"
									__lowercase					=				tgt_lang
									__lowercase					=				self.get_lang_id(self._src_lang   )
									self.set_src_lang_special_tokens(self._src_lang   )

									__lowercase					=				num_madeup_words





			@property
			def 		snake_case__			(				self	:       str   )						->					int:
									"""simple docstring"""







									return len(self.encoder   ) + len(self.lang_token_to_id   )





			@property
			def 		snake_case__			(				self	:       Any   )						->					str:
									"""simple docstring"""







									return self._src_lang





			@src_lang.setter
			def 		snake_case__			(				self	:       List[str]			,     lowercase	:       str   )						->					None:
									"""simple docstring"""







									__lowercase					=				new_src_lang
									self.set_src_lang_special_tokens(self._src_lang   )





			def 		snake_case__			(				self	:       Union[str, Any]			,     lowercase	:       str   )						->					List[str]:
									"""simple docstring"""







									return self.sp_model.encode(__lowerCamelCase			,     out_type=__lowerCamelCase   )





			def 		snake_case__			(				self	:       List[str]			,     lowercase	:       str   )						->					Tuple:
									"""simple docstring"""







									if token in self.lang_token_to_id:
															return self.lang_token_to_id[token]
									return self.encoder.get(__lowerCamelCase			,     self.encoder[self.unk_token]   )





			def 		snake_case__			(				self	:       Union[str, Any]			,     lowercase	:       int   )						->					str:
									"""simple docstring"""







									if index in self.id_to_lang_token:
															return self.id_to_lang_token[index]
									return self.decoder.get(__lowerCamelCase			,     self.unk_token   )





			def 		snake_case__			(				self	:       Optional[int]			,     lowercase	:       Optional[Any]   )						->					List[Any]:
									"""simple docstring"""







									__lowercase					=				[]
									__lowercase					=				""
									for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																					out_string += self.sp_model.decode(__lowerCamelCase   ) + token
																					__lowercase					=				[]
															else:
																					current_sub_tokens.append(__lowerCamelCase   )
									out_string += self.sp_model.decode(__lowerCamelCase   )
									return out_string.strip()





			def 		snake_case__			(				self	:       int			,     lowercase	:       List[int]			,     lowercase	:       Optional[List[int]] = None			,     lowercase	:       bool = False   )						->					List[int]:
									"""simple docstring"""







									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=__lowerCamelCase			,     token_ids_a=__lowerCamelCase			,     already_has_special_tokens=__lowerCamelCase   )

									__lowercase					=				[1] * len(self.prefix_tokens   )
									__lowercase					=				[1] * len(self.suffix_tokens   )
									if token_ids_a is None:
															return prefix_ones + ([0] * len(__lowerCamelCase   )) + suffix_ones
									return prefix_ones + ([0] * len(__lowerCamelCase   )) + ([0] * len(__lowerCamelCase   )) + suffix_ones





			def 		snake_case__			(				self	:       Union[str, Any]			,     lowercase	:       List[int]			,     lowercase	:       Optional[List[int]] = None   )						->					List[int]:
									"""simple docstring"""







									if token_ids_a is None:
															return self.prefix_tokens + token_ids_a + self.suffix_tokens
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





			def 		snake_case__			(				self	:       List[str]   )						->					Dict:
									"""simple docstring"""







									__lowercase					=				{self.convert_ids_to_tokens(__lowerCamelCase   ): i for i in range(self.vocab_size   )}
									vocab.update(self.added_tokens_encoder   )
									return vocab





			def __getstate__(				self	:       Optional[int]   )						->					Dict:
									"""simple docstring"""







									__lowercase					=				self.__dict__.copy()
									__lowercase					=				None
									return state





			def __setstate__(				self	:       str			,     lowercase	:       Dict   )						->					None:
									"""simple docstring"""







									__lowercase					=				d

									# for backward compatibility
									if not hasattr(self			,     """sp_model_kwargs"""   ):
															__lowercase					=				{}

									__lowercase					=				load_spm(self.spm_file			,     self.sp_model_kwargs   )





			def 		snake_case__			(				self	:       List[str]			,     lowercase	:       str			,     lowercase	:       Optional[str] = None   )						->					Tuple[str]:
									"""simple docstring"""







									__lowercase					=				Path(__lowerCamelCase   )
									if not save_dir.is_dir():
															raise OSError(F"{save_directory} should be a directory"   )
									__lowercase					=				save_dir / (
									    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
									)
									__lowercase					=				save_dir / (
									    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
									)

									save_json(self.encoder			,     __lowerCamelCase   )

									if os.path.abspath(self.spm_file   ) != os.path.abspath(__lowerCamelCase   ) and os.path.isfile(self.spm_file   ):
															copyfile(self.spm_file			,     __lowerCamelCase   )
									elif not os.path.isfile(self.spm_file   ):
															with open(__lowerCamelCase			,     """wb"""   ) as fi:
																					__lowercase					=				self.sp_model.serialized_model_proto()
																					fi.write(__lowerCamelCase   )

									return (str(__lowerCamelCase   ), str(__lowerCamelCase   ))





			def 		snake_case__			(				self	:       Union[str, Any]			,     lowercase	:       List[str]			,     lowercase	:       str = "en"			,     lowercase	:       Optional[List[str]] = None			,     lowercase	:       str = "ro"			,     **lowercase	:       Tuple			,     )						->					BatchEncoding:
									"""simple docstring"""







									__lowercase					=				src_lang
									__lowercase					=				tgt_lang
									self.set_src_lang_special_tokens(self.src_lang   )
									return super().prepare_seqaseq_batch(__lowerCamelCase			,     __lowerCamelCase			,     **__lowerCamelCase   )





			def 		snake_case__			(				self	:       Union[str, Any]			,     lowercase	:       Dict			,     lowercase	:       Optional[str]			,     lowercase	:       Optional[str]			,     **lowercase	:       Dict   )						->					Optional[int]:
									"""simple docstring"""







									if src_lang is None or tgt_lang is None:
															raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""   )
									__lowercase					=				src_lang
									__lowercase					=				self(__lowerCamelCase			,     add_special_tokens=__lowerCamelCase			,     **__lowerCamelCase   )
									__lowercase					=				self.get_lang_id(__lowerCamelCase   )
									__lowercase					=				tgt_lang_id
									return inputs





			def 		snake_case__			(				self	:       Dict   )						->					Optional[int]:
									"""simple docstring"""







									self.set_src_lang_special_tokens(self.src_lang   )





			def 		snake_case__			(				self	:       Dict   )						->					Optional[int]:
									"""simple docstring"""







									self.set_tgt_lang_special_tokens(self.tgt_lang   )





			def 		snake_case__			(				self	:       Tuple			,     lowercase	:       str   )						->					None:
									"""simple docstring"""







									__lowercase					=				self.get_lang_token(__lowerCamelCase   )
									__lowercase					=				self.lang_token_to_id[lang_token]
									__lowercase					=				[self.cur_lang_id]
									__lowercase					=				[self.eos_token_id]





			def 		snake_case__			(				self	:       Tuple			,     lowercase	:       str   )						->					None:
									"""simple docstring"""







									__lowercase					=				self.get_lang_token(__lowerCamelCase   )
									__lowercase					=				self.lang_token_to_id[lang_token]
									__lowercase					=				[self.cur_lang_id]
									__lowercase					=				[self.eos_token_id]





			def 		snake_case__			(				self	:       List[Any]			,     lowercase	:       str   )						->					str:
									"""simple docstring"""







									return self.lang_code_to_token[lang]







			def 		snake_case__			(				self	:       List[str]			,     lowercase	:       str   )						->					int:
									"""simple docstring"""







									__lowercase					=				self.get_lang_token(__lowerCamelCase   )
									return self.lang_token_to_id[lang_token]






def  UpperCAmelCase__    (    lowercase__						,       lowercase__   ) ->      Any:
						__lowercase					=				sentencepiece.SentencePieceProcessor(**_lowerCamelCase   )
						spm.Load(str(_lowerCamelCase   )   )
						return spm






def  UpperCAmelCase__    (    lowercase__   ) ->      Optional[Any]:
						with open(_lowerCamelCase						,       """r"""   ) as f:
												return json.load(_lowerCamelCase   )







def  UpperCAmelCase__    (    lowercase__						,       lowercase__   ) ->      Union[str, Any]:
						with open(_lowerCamelCase						,       """w"""   ) as f:
												json.dump(_lowerCamelCase						,       _lowerCamelCase						,       indent=2   )










def  UpperCAmelCase__    (    lowercase__   ) ->      Optional[int]:
						__lowercase					=				len(lowercase__   )
						__lowercase					=				sum(lowercase__   )

						__lowercase					=				[[False for x in range(s + 1   )] for y in range(n + 1   )]

						for i in range(1						,       n + 1   ):
												__lowercase					=				True

						for i in range(1						,       s + 1   ):
												__lowercase					=				False

						for i in range(1						,       n + 1   ):
												for j in range(1						,       s + 1   ):
																		__lowercase					=				dp[i][j - 1]

																		if arr[i - 1] <= j:
																								__lowercase					=				dp[i][j] or dp[i - 1][j - arr[i - 1]]

						for j in range(int(s / 2   )						,       -1						,       -1   ):
												if dp[n][j] is True:
																		__lowercase					=				s - 2 * j
																		break

						return diff





