


'''simple docstring'''




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase  :		List[str]		     =	logging.get_logger(__name__)

_lowerCAmelCase  :		Optional[Any]		     =	'▁'

_lowerCAmelCase  :		List[Any]		     =	{'vocab_file': 'sentencepiece.bpe.model'}

_lowerCAmelCase  :		List[Any]		     =	{
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

_lowerCAmelCase  :		Union[str, Any]		     =	{
    'facebook/mbart-large-50-one-to-many-mmt': 1024,
}

# fmt: off
_lowerCAmelCase  :		Tuple		     =	['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']



class  lowerCAmelCase				(				a							):
	_lowerCamelCase   :					str			      = VOCAB_FILES_NAMES
	_lowerCamelCase   :					int			      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowerCamelCase   :					Tuple			      = PRETRAINED_VOCAB_FILES_MAP
	_lowerCamelCase   :					Union[str, Any]			      = ["""input_ids""", """attention_mask"""]

	_lowerCamelCase   :					List[int]			      = []
	_lowerCamelCase   :					List[int]			      = []



	def __init__(					self ,					snake_case__ ,					snake_case__=None ,					snake_case__=None ,					snake_case__="</s>" ,					snake_case__="</s>" ,					snake_case__="<s>" ,					snake_case__="<unk>" ,					snake_case__="<pad>" ,					snake_case__="<mask>" ,					snake_case__ = None ,					**snake_case__ ,					):
		# Mask token behave like a normal word, i.e. include the space before it
		lowerCAmelCase					:    Tuple      							= AddedToken(snake_case__ ,					lstrip=snake_case__ ,					rstrip=snake_case__    ) if isinstance(snake_case__ ,					snake_case__    ) else mask_token

		lowerCAmelCase					:    List[str]      							= {} if sp_model_kwargs is None else sp_model_kwargs

		lowerCAmelCase					:    List[Any]      							= kwargs.get('additional_special_tokens' ,					[]    )
		kwargs["additional_special_tokens"] += [
		    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
		]

		super().__init__(
		    src_lang=snake_case__ ,					tgt_lang=snake_case__ ,					eos_token=snake_case__ ,					unk_token=snake_case__ ,					sep_token=snake_case__ ,					cls_token=snake_case__ ,					pad_token=snake_case__ ,					mask_token=snake_case__ ,					sp_model_kwargs=self.sp_model_kwargs ,					**snake_case__ ,					)

		lowerCAmelCase					:    int      							= spm.SentencePieceProcessor(**self.sp_model_kwargs    )
		self.sp_model.Load(str(snake_case__    )    )
		lowerCAmelCase					:    int      							= vocab_file

		# Original fairseq vocab and spm vocab must be "aligned":
		# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
		# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
		# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
		# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

		# Mimic fairseq token-to-id alignment for the first 4 token
		lowerCAmelCase					:    Any      							= {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

		# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
		lowerCAmelCase					:    List[str]      							= 1

		lowerCAmelCase					:    str      							= len(self.sp_model    )
		lowerCAmelCase					:    Union[str, Any]      							= {
		    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(snake_case__    )
		}
		lowerCAmelCase					:    int      							= {v: k for k, v in self.lang_code_to_id.items()}
		lowerCAmelCase					:    Union[str, Any]      							= len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset

		self.fairseq_tokens_to_ids.update(self.lang_code_to_id    )
		lowerCAmelCase					:    Dict      							= {v: k for k, v in self.fairseq_tokens_to_ids.items()}

		lowerCAmelCase					:    List[Any]      							= src_lang if src_lang is not None else 'en_XX'
		lowerCAmelCase					:    Optional[Any]      							= self.lang_code_to_id[self._src_lang]
		lowerCAmelCase					:    Union[str, Any]      							= tgt_lang
		self.set_src_lang_special_tokens(self._src_lang    )



	@property
	def 				lowercase     (					self    ):
		return len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset + 1  # Plus 1 for the mask token



	@property
	def 				lowercase     (					self    ):
		return self._src_lang



	@src_lang.setter
	def 				lowercase     (					self ,					snake_case__    ):
		lowerCAmelCase					:    Optional[Any]      							= new_src_lang
		self.set_src_lang_special_tokens(self._src_lang    )



	def __getstate__(					self    ):
		lowerCAmelCase					:    List[Any]      							= self.__dict__.copy()
		lowerCAmelCase					:    Union[str, Any]      							= None
		return state



	def __setstate__(					self ,					snake_case__    ):
		lowerCAmelCase					:    List[str]      							= d

		# for backward compatibility
		if not hasattr(self ,					'sp_model_kwargs'    ):
			lowerCAmelCase					:    List[str]      							= {}

		lowerCAmelCase					:    Any      							= spm.SentencePieceProcessor(**self.sp_model_kwargs    )
		self.sp_model.Load(self.vocab_file    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Any      							= {self.convert_ids_to_tokens(snake_case__    ): i for i in range(self.vocab_size    )}
		vocab.update(self.added_tokens_encoder    )
		return vocab



	def 				lowercase     (					self ,					snake_case__    ):
		return self.sp_model.encode(snake_case__ ,					out_type=snake_case__    )



	def 				lowercase     (					self ,					snake_case__    ):
		if token in self.fairseq_tokens_to_ids:
			return self.fairseq_tokens_to_ids[token]
		lowerCAmelCase					:    Union[str, Any]      							= self.sp_model.PieceToId(snake_case__    )

		# Need to return unknown token if the SP model returned 0
		return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



	def 				lowercase     (					self ,					snake_case__    ):
		if index in self.fairseq_ids_to_tokens:
			return self.fairseq_ids_to_tokens[index]
		return self.sp_model.IdToPiece(index - self.fairseq_offset    )



	def 				lowercase     (					self ,					snake_case__    ):
		lowerCAmelCase					:    Union[str, Any]      							= []
		lowerCAmelCase					:    Any      							= ''
		lowerCAmelCase					:    Optional[Any]      							= False
		for token in tokens:
			# make sure that special tokens are not decoded using sentencepiece model
			if token in self.all_special_tokens:
				if not prev_is_special:
					out_string += " "
				out_string += self.sp_model.decode(snake_case__    ) + token
				lowerCAmelCase					:    Optional[int]      							= True
				lowerCAmelCase					:    Tuple      							= []
			else:
				current_sub_tokens.append(snake_case__    )
				lowerCAmelCase					:    str      							= False
		out_string += self.sp_model.decode(snake_case__    )
		return out_string.strip()



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ = None    ):
		if not os.path.isdir(snake_case__    ):
			logger.error(f"Vocabulary path ({save_directory}) should be a directory"    )
			return
		lowerCAmelCase					:    Dict      							= os.path.join(
		    snake_case__ ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

		if os.path.abspath(self.vocab_file    ) != os.path.abspath(snake_case__    ) and os.path.isfile(self.vocab_file    ):
			copyfile(self.vocab_file ,					snake_case__    )
		elif not os.path.isfile(self.vocab_file    ):
			with open(snake_case__ ,					'wb'    ) as fi:
				lowerCAmelCase					:    str      							= self.sp_model.serialized_model_proto()
				fi.write(snake_case__    )

		return (out_vocab_file,)



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ = None ,					snake_case__ = False    ):

		if already_has_special_tokens:
			return super().get_special_tokens_mask(
			    token_ids_a=snake_case__ ,					token_ids_a=snake_case__ ,					already_has_special_tokens=snake_case__    )

		lowerCAmelCase					:    List[Any]      							= [1] * len(self.prefix_tokens    )
		lowerCAmelCase					:    Tuple      							= [1] * len(self.suffix_tokens    )
		if token_ids_a is None:
			return prefix_ones + ([0] * len(snake_case__    )) + suffix_ones
		return prefix_ones + ([0] * len(snake_case__    )) + ([0] * len(snake_case__    )) + suffix_ones



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ = None    ):
		if token_ids_a is None:
			return self.prefix_tokens + token_ids_a + self.suffix_tokens
		# We don't expect to process pairs, but leave the pair logic for API consistency
		return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					**snake_case__    ):
		if src_lang is None or tgt_lang is None:
			raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'    )
		lowerCAmelCase					:    Optional[Any]      							= src_lang
		lowerCAmelCase					:    Optional[Any]      							= self(snake_case__ ,					add_special_tokens=snake_case__ ,					return_tensors=snake_case__ ,					**snake_case__    )
		lowerCAmelCase					:    Dict      							= self.convert_tokens_to_ids(snake_case__    )
		lowerCAmelCase					:    Optional[Any]      							= tgt_lang_id
		return inputs



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ = "en_XX" ,					snake_case__ = None ,					snake_case__ = "ro_RO" ,					**snake_case__ ,					):
		lowerCAmelCase					:    List[str]      							= src_lang
		lowerCAmelCase					:    Any      							= tgt_lang
		return super().prepare_seqaseq_batch(snake_case__ ,					snake_case__ ,					**snake_case__    )



	def 				lowercase     (					self    ):
		return self.set_src_lang_special_tokens(self.src_lang    )



	def 				lowercase     (					self    ):
		return self.set_tgt_lang_special_tokens(self.tgt_lang    )



	def 				lowercase     (					self ,					snake_case__    ):
		lowerCAmelCase					:    Optional[Any]      							= self.lang_code_to_id[src_lang]
		lowerCAmelCase					:    Tuple      							= [self.cur_lang_code_id]
		lowerCAmelCase					:    List[Any]      							= [self.eos_token_id]



	def 				lowercase     (					self ,					snake_case__    ):
		lowerCAmelCase					:    List[Any]      							= self.lang_code_to_id[tgt_lang]
		lowerCAmelCase					:    Tuple      							= [self.cur_lang_code_id]
		lowerCAmelCase					:    Optional[int]      							= [self.eos_token_id]




'''simple docstring'''




def 			__UpperCamelCase    (    _A    :				int				)    ->      bool:




	"""simple docstring"""





	return number & 1 == 0


if __name__ == "__main__":
				import doctest

				doctest.testmod()
