





'''simple docstring'''





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_						   =    logging.get_logger(__name__)

lowerCAmelCase_						   =    '''▁'''

lowerCAmelCase_						   =    {'''vocab_file''': '''sentencepiece.bpe.model'''}

lowerCAmelCase_						   =    {
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

lowerCAmelCase_						   =    {
    '''facebook/mbart-large-50-one-to-many-mmt''': 1024,
}

# fmt: off
lowerCAmelCase_						   =    ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']



class        _lowerCAmelCase    (			UpperCamelCase__							):


  '''simple docstring'''
  a_							:       Union[str, Any]      					=VOCAB_FILES_NAMES
  a_							:       Any      					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_							:       Optional[int]      					=PRETRAINED_VOCAB_FILES_MAP
  a_							:       List[str]      					=["""input_ids""", """attention_mask"""]

  a_							:       List[int]      					=[]
  a_							:       List[int]      					=[]







  def __init__(      self      :					Tuple      ,				UpperCamelCase      :					List[str]      ,				UpperCamelCase      :					int=None      ,				UpperCamelCase      :					List[str]=None      ,				UpperCamelCase      :					Dict="</s>"      ,				UpperCamelCase      :					Union[str, Any]="</s>"      ,				UpperCamelCase      :					Dict="<s>"      ,				UpperCamelCase      :					Tuple="<unk>"      ,				UpperCamelCase      :					Union[str, Any]="<pad>"      ,				UpperCamelCase      :					Optional[Any]="<mask>"      ,				UpperCamelCase      :					Dict = None      ,				**UpperCamelCase      :					Optional[Any]      ,				):


    '''simple docstring'''





    _snake_case   :				int														=	AddedToken(UpperCamelCase      ,				lstrip=UpperCamelCase      ,				rstrip=UpperCamelCase							) if isinstance(UpperCamelCase      ,				UpperCamelCase							) else mask_token

    _snake_case   :				Union[str, Any]														=	{} if sp_model_kwargs is None else sp_model_kwargs

    _snake_case   :				str														=	kwargs.get('additional_special_tokens'      ,				[]							)
    kwargs["additional_special_tokens"] += [
        code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
    ]

    super().__init__(
        src_lang=UpperCamelCase      ,				tgt_lang=UpperCamelCase      ,				eos_token=UpperCamelCase      ,				unk_token=UpperCamelCase      ,				sep_token=UpperCamelCase      ,				cls_token=UpperCamelCase      ,				pad_token=UpperCamelCase      ,				mask_token=UpperCamelCase      ,				sp_model_kwargs=self.sp_model_kwargs      ,				**UpperCamelCase      ,				)

    _snake_case   :				Optional[int]														=	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
    self.sp_model.Load(str(UpperCamelCase							)							)
    _snake_case   :				Any														=	vocab_file

    # Original fairseq vocab and spm vocab must be "aligned":
    # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
    # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
    # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
    # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

    # Mimic fairseq token-to-id alignment for the first 4 token
    _snake_case   :				Optional[Any]														=	{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

    # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
    _snake_case   :				str														=	1

    _snake_case   :				Union[str, Any]														=	len(self.sp_model							)
    _snake_case   :				Optional[int]														=	{
        code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(UpperCamelCase							)
    }
    _snake_case   :				Union[str, Any]														=	{v: k for k, v in self.lang_code_to_id.items()}
    _snake_case   :				Tuple														=	len(self.sp_model							) + len(self.lang_code_to_id							) + self.fairseq_offset

    self.fairseq_tokens_to_ids.update(self.lang_code_to_id							)
    _snake_case   :				Optional[int]														=	{v: k for k, v in self.fairseq_tokens_to_ids.items()}

    _snake_case   :				Tuple														=	src_lang if src_lang is not None else 'en_XX'
    _snake_case   :				List[str]														=	self.lang_code_to_id[self._src_lang]
    _snake_case   :				List[Any]														=	tgt_lang
    self.set_src_lang_special_tokens(self._src_lang							)







  @property
  def UpperCamelCase_ (      self      :					Any							):


    '''simple docstring'''





    return len(self.sp_model							) + len(self.lang_code_to_id							) + self.fairseq_offset + 1  # Plus 1 for the mask token







  @property
  def UpperCamelCase_ (      self      :					List[str]							):


    '''simple docstring'''





    return self._src_lang







  @src_lang.setter
  def UpperCamelCase_ (      self      :					str      ,				UpperCamelCase      :					Any							):


    '''simple docstring'''





    _snake_case   :				Optional[Any]														=	new_src_lang
    self.set_src_lang_special_tokens(self._src_lang							)







  def __getstate__(      self      :					str							):


    '''simple docstring'''





    _snake_case   :				List[Any]														=	self.__dict__.copy()
    _snake_case   :				Optional[Any]														=	None
    return state







  def __setstate__(      self      :					Optional[Any]      ,				UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				List[str]														=	d

    # for backward compatibility
    if not hasattr(self      ,				'sp_model_kwargs'							):
      _snake_case   :				Optional[Any]														=	{}

    _snake_case   :				str														=	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
    self.sp_model.Load(self.vocab_file							)







  def UpperCamelCase_ (      self      :					Optional[int]							):


    '''simple docstring'''





    _snake_case   :				int														=	{self.convert_ids_to_tokens(UpperCamelCase							): i for i in range(self.vocab_size							)}
    vocab.update(self.added_tokens_encoder							)
    return vocab







  def UpperCamelCase_ (      self      :					Optional[Any]      ,				UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    return self.sp_model.encode(UpperCamelCase      ,				out_type=UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Tuple      ,				UpperCamelCase      :					Optional[Any]							):


    '''simple docstring'''





    if token in self.fairseq_tokens_to_ids:
      return self.fairseq_tokens_to_ids[token]
    _snake_case   :				Optional[int]														=	self.sp_model.PieceToId(UpperCamelCase							)

    # Need to return unknown token if the SP model returned 0
    return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







  def UpperCamelCase_ (      self      :					List[Any]      ,				UpperCamelCase      :					Tuple							):


    '''simple docstring'''





    if index in self.fairseq_ids_to_tokens:
      return self.fairseq_ids_to_tokens[index]
    return self.sp_model.IdToPiece(index - self.fairseq_offset							)







  def UpperCamelCase_ (      self      :					Optional[int]      ,				UpperCamelCase      :					Dict							):


    '''simple docstring'''





    _snake_case   :				Optional[int]														=	[]
    _snake_case   :				Optional[int]														=	''
    _snake_case   :				Optional[int]														=	False
    for token in tokens:
      # make sure that special tokens are not decoded using sentencepiece model
      if token in self.all_special_tokens:
        if not prev_is_special:
          out_string += " "
        out_string += self.sp_model.decode(UpperCamelCase							) + token
        _snake_case   :				Union[str, Any]														=	True
        _snake_case   :				int														=	[]
      else:
        current_sub_tokens.append(UpperCamelCase							)
        _snake_case   :				Union[str, Any]														=	False
    out_string += self.sp_model.decode(UpperCamelCase							)
    return out_string.strip()







  def UpperCamelCase_ (      self      :					List[str]      ,				UpperCamelCase      :					Tuple      ,				UpperCamelCase      :					List[Any] = None							):


    '''simple docstring'''





    if not os.path.isdir(UpperCamelCase							):
      logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
      return
    _snake_case   :				Tuple														=	os.path.join(
        UpperCamelCase      ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

    if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCamelCase							) and os.path.isfile(self.vocab_file							):
      copyfile(self.vocab_file      ,				UpperCamelCase							)
    elif not os.path.isfile(self.vocab_file							):
      with open(UpperCamelCase      ,				'wb'							) as fi:
        _snake_case   :				Union[str, Any]														=	self.sp_model.serialized_model_proto()
        fi.write(UpperCamelCase							)

    return (out_vocab_file,)







  def UpperCamelCase_ (      self      :					List[Any]      ,				UpperCamelCase      :					List[Any]      ,				UpperCamelCase      :					Tuple = None      ,				UpperCamelCase      :					Optional[int] = False							):


    '''simple docstring'''





    if already_has_special_tokens:
      return super().get_special_tokens_mask(
          token_ids_a=UpperCamelCase      ,				token_ids_a=UpperCamelCase      ,				already_has_special_tokens=UpperCamelCase							)

    _snake_case   :				Dict														=	[1] * len(self.prefix_tokens							)
    _snake_case   :				str														=	[1] * len(self.suffix_tokens							)
    if token_ids_a is None:
      return prefix_ones + ([0] * len(UpperCamelCase							)) + suffix_ones
    return prefix_ones + ([0] * len(UpperCamelCase							)) + ([0] * len(UpperCamelCase							)) + suffix_ones







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					Any      ,				UpperCamelCase      :					List[str] = None							):


    '''simple docstring'''





    if token_ids_a is None:
      return self.prefix_tokens + token_ids_a + self.suffix_tokens
    # We don't expect to process pairs, but leave the pair logic for API consistency
    return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







  def UpperCamelCase_ (      self      :					int      ,				UpperCamelCase      :					Dict      ,				UpperCamelCase      :					Dict      ,				UpperCamelCase      :					Optional[Any]      ,				UpperCamelCase      :					Optional[int]      ,				**UpperCamelCase      :					Tuple							):


    '''simple docstring'''





    if src_lang is None or tgt_lang is None:
      raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'							)
    _snake_case   :				Dict														=	src_lang
    _snake_case   :				str														=	self(UpperCamelCase      ,				add_special_tokens=UpperCamelCase      ,				return_tensors=UpperCamelCase      ,				**UpperCamelCase							)
    _snake_case   :				Optional[Any]														=	self.convert_tokens_to_ids(UpperCamelCase							)
    _snake_case   :				Tuple														=	tgt_lang_id
    return inputs







  def UpperCamelCase_ (      self      :					Optional[int]      ,				UpperCamelCase      :					str      ,				UpperCamelCase      :					str = "en_XX"      ,				UpperCamelCase      :					Tuple = None      ,				UpperCamelCase      :					Any = "ro_RO"      ,				**UpperCamelCase      :					Optional[int]      ,				):


    '''simple docstring'''





    _snake_case   :				int														=	src_lang
    _snake_case   :				int														=	tgt_lang
    return super().prepare_seqaseq_batch(UpperCamelCase      ,				UpperCamelCase      ,				**UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Tuple							):


    '''simple docstring'''





    return self.set_src_lang_special_tokens(self.src_lang							)







  def UpperCamelCase_ (      self      :					str							):


    '''simple docstring'''





    return self.set_tgt_lang_special_tokens(self.tgt_lang							)







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					List[str]							):


    '''simple docstring'''





    _snake_case   :				Optional[Any]														=	self.lang_code_to_id[src_lang]
    _snake_case   :				Optional[int]														=	[self.cur_lang_code_id]
    _snake_case   :				List[str]														=	[self.eos_token_id]







  def UpperCamelCase_ (      self      :					Dict      ,				UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				Optional[int]														=	self.lang_code_to_id[tgt_lang]
    _snake_case   :				Optional[Any]														=	[self.cur_lang_code_id]
    _snake_case   :				Optional[Any]														=	[self.eos_token_id]











def     lowerCamelCase_	(      lowerCAmelCase:	int				)->		int:

  if not isinstance(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				Union[str, Any]														=	F"""Input value of [number={number}] must be an integer"""
    raise TypeError(lowerCAmelCase				)

  if number < 1:
    _snake_case   :				int														=	F"""Input value of [number={number}] must be > 0"""
    raise ValueError(lowerCAmelCase				)

  _snake_case   :				int														=	1

  for i in range(1    ,							lowerCAmelCase				):
    current_number *= 4 * i - 2
    current_number //= i + 1

  return current_number


if __name__ == "__main__":
      import doctest

      doctest.testmod()




