


'''simple docstring'''







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase    :     Optional[Any]					=			logging.get_logger(__name__)

lowerCAmelCase    :     Tuple					=			'▁'

lowerCAmelCase    :     Optional[Any]					=			{'vocab_file': 'sentencepiece.bpe.model'}

lowerCAmelCase    :     str					=			{
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

lowerCAmelCase    :     List[str]					=			{
    'facebook/mbart-large-50-one-to-many-mmt': 10_24,
}

# fmt: off
lowerCAmelCase    :     Union[str, Any]					=			['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']
class 	UpperCamelCase__					( UpperCamelCase_	):



     """simple docstring"""






     __magic_name__    			=    VOCAB_FILES_NAMES
     __magic_name__    			=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__    			=    PRETRAINED_VOCAB_FILES_MAP
     __magic_name__    			=    ["input_ids", "attention_mask"]

     __magic_name__    			=    []
     __magic_name__    			=    []







     def __init__( self      ,	snake_case__      ,	snake_case__=None      ,	snake_case__=None      ,	snake_case__="</s>"      ,	snake_case__="</s>"      ,	snake_case__="<s>"      ,	snake_case__="<unk>"      ,	snake_case__="<pad>"      ,	snake_case__="<mask>"      ,	snake_case__ = None      ,	**snake_case__      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							Dict   			=       AddedToken(snake_case__      ,	lstrip=snake_case__      ,	rstrip=snake_case__   ) if isinstance(snake_case__      ,	snake_case__   ) else mask_token

            _lowerCAmelCase    :							Dict   			=       {} if sp_model_kwargs is None else sp_model_kwargs

            _lowerCAmelCase    :							Dict   			=       kwargs.get('additional_special_tokens'      ,	[]   )
            kwargs["additional_special_tokens"] += [
                code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
            ]

            super().__init__(
                src_lang=snake_case__      ,	tgt_lang=snake_case__      ,	eos_token=snake_case__      ,	unk_token=snake_case__      ,	sep_token=snake_case__      ,	cls_token=snake_case__      ,	pad_token=snake_case__      ,	mask_token=snake_case__      ,	sp_model_kwargs=self.sp_model_kwargs      ,	**snake_case__      ,	)

            _lowerCAmelCase    :							Optional[int]   			=       spm.SentencePieceProcessor(**self.sp_model_kwargs   )
            self.sp_model.Load(str(snake_case__   )   )
            _lowerCAmelCase    :							int   			=       vocab_file

            # Original fairseq vocab and spm vocab must be "aligned":
            # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
            # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
            # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
            # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

            # Mimic fairseq token-to-id alignment for the first 4 token
            _lowerCAmelCase    :							List[str]   			=       {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

            # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
            _lowerCAmelCase    :							int   			=       1

            _lowerCAmelCase    :							Dict   			=       len(self.sp_model   )
            _lowerCAmelCase    :							int   			=       {
                code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(snake_case__   )
            }
            _lowerCAmelCase    :							int   			=       {v: k for k, v in self.lang_code_to_id.items()}
            _lowerCAmelCase    :							int   			=       len(self.sp_model   ) + len(self.lang_code_to_id   ) + self.fairseq_offset

            self.fairseq_tokens_to_ids.update(self.lang_code_to_id   )
            _lowerCAmelCase    :							List[Any]   			=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}

            _lowerCAmelCase    :							Optional[Any]   			=       src_lang if src_lang is not None else '''en_XX'''
            _lowerCAmelCase    :							Tuple   			=       self.lang_code_to_id[self._src_lang]
            _lowerCAmelCase    :							List[str]   			=       tgt_lang
            self.set_src_lang_special_tokens(self._src_lang   )







     @property
     def a	( self   ):

            '''simple docstring'''

            return len(self.sp_model   ) + len(self.lang_code_to_id   ) + self.fairseq_offset + 1  # Plus 1 for the mask token







     @property
     def a	( self   ):

            '''simple docstring'''

            return self._src_lang







     @src_lang.setter
     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Tuple   			=       new_src_lang
            self.set_src_lang_special_tokens(self._src_lang   )







     def __getstate__( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Tuple   			=       self.__dict__.copy()
            _lowerCAmelCase    :							Dict   			=       None
            return state







     def __setstate__( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Dict   			=       d

            # for backward compatibility
            if not hasattr(self      ,	'sp_model_kwargs'   ):
                   _lowerCAmelCase    :							List[str]   			=       {}

            _lowerCAmelCase    :							Tuple   			=       spm.SentencePieceProcessor(**self.sp_model_kwargs   )
            self.sp_model.Load(self.vocab_file   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[Any]   			=       {self.convert_ids_to_tokens(snake_case__   ): i for i in range(self.vocab_size   )}
            vocab.update(self.added_tokens_encoder   )
            return vocab







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            return self.sp_model.encode(snake_case__      ,	out_type=snake_case__   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            if token in self.fairseq_tokens_to_ids:
                   return self.fairseq_tokens_to_ids[token]
            _lowerCAmelCase    :							int   			=       self.sp_model.PieceToId(snake_case__   )

            # Need to return unknown token if the SP model returned 0
            return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            if index in self.fairseq_ids_to_tokens:
                   return self.fairseq_ids_to_tokens[index]
            return self.sp_model.IdToPiece(index - self.fairseq_offset   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Any   			=       []
            _lowerCAmelCase    :							Any   			=       ''''''
            _lowerCAmelCase    :							Dict   			=       False
            for token in tokens:
                   # make sure that special tokens are not decoded using sentencepiece model
                   if token in self.all_special_tokens:
                          if not prev_is_special:
                                 out_string += " "
                          out_string += self.sp_model.decode(snake_case__   ) + token
                          _lowerCAmelCase    :							Tuple   			=       True
                          _lowerCAmelCase    :							Optional[int]   			=       []
                   else:
                          current_sub_tokens.append(snake_case__   )
                          _lowerCAmelCase    :							Tuple   			=       False
            out_string += self.sp_model.decode(snake_case__   )
            return out_string.strip()







     def a	( self      ,	snake_case__      ,	snake_case__ = None   ):

            '''simple docstring'''

            if not os.path.isdir(snake_case__   ):
                   logger.error(F'Vocabulary path ({save_directory}) should be a directory'   )
                   return
            _lowerCAmelCase    :							Dict   			=       os.path.join(
                snake_case__      ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

            if os.path.abspath(self.vocab_file   ) != os.path.abspath(snake_case__   ) and os.path.isfile(self.vocab_file   ):
                   copyfile(self.vocab_file      ,	snake_case__   )
            elif not os.path.isfile(self.vocab_file   ):
                   with open(snake_case__      ,	'wb'   ) as fi:
                          _lowerCAmelCase    :							Tuple   			=       self.sp_model.serialized_model_proto()
                          fi.write(snake_case__   )

            return (out_vocab_file,)







     def a	( self      ,	snake_case__      ,	snake_case__ = None      ,	snake_case__ = False   ):

            '''simple docstring'''

            if already_has_special_tokens:
                   return super().get_special_tokens_mask(
                       token_ids_a=snake_case__      ,	token_ids_a=snake_case__      ,	already_has_special_tokens=snake_case__   )

            _lowerCAmelCase    :							Optional[Any]   			=       [1] * len(self.prefix_tokens   )
            _lowerCAmelCase    :							Union[str, Any]   			=       [1] * len(self.suffix_tokens   )
            if token_ids_a is None:
                   return prefix_ones + ([0] * len(snake_case__   )) + suffix_ones
            return prefix_ones + ([0] * len(snake_case__   )) + ([0] * len(snake_case__   )) + suffix_ones







     def a	( self      ,	snake_case__      ,	snake_case__ = None   ):

            '''simple docstring'''

            if token_ids_a is None:
                   return self.prefix_tokens + token_ids_a + self.suffix_tokens
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







     def a	( self      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	**snake_case__   ):

            '''simple docstring'''

            if src_lang is None or tgt_lang is None:
                   raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'   )
            _lowerCAmelCase    :							Dict   			=       src_lang
            _lowerCAmelCase    :							Union[str, Any]   			=       self(snake_case__      ,	add_special_tokens=snake_case__      ,	return_tensors=snake_case__      ,	**snake_case__   )
            _lowerCAmelCase    :							Optional[Any]   			=       self.convert_tokens_to_ids(snake_case__   )
            _lowerCAmelCase    :							Optional[int]   			=       tgt_lang_id
            return inputs







     def a	( self      ,	snake_case__      ,	snake_case__ = "en_XX"      ,	snake_case__ = None      ,	snake_case__ = "ro_RO"      ,	**snake_case__      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							str   			=       src_lang
            _lowerCAmelCase    :							int   			=       tgt_lang
            return super().prepare_seqaseq_batch(snake_case__      ,	snake_case__      ,	**snake_case__   )







     def a	( self   ):

            '''simple docstring'''

            return self.set_src_lang_special_tokens(self.src_lang   )







     def a	( self   ):

            '''simple docstring'''

            return self.set_tgt_lang_special_tokens(self.tgt_lang   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							List[str]   			=       self.lang_code_to_id[src_lang]
            _lowerCAmelCase    :							Dict   			=       [self.cur_lang_code_id]
            _lowerCAmelCase    :							Union[str, Any]   			=       [self.eos_token_id]







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Any   			=       self.lang_code_to_id[tgt_lang]
            _lowerCAmelCase    :							Tuple   			=       [self.cur_lang_code_id]
            _lowerCAmelCase    :							List[Any]   			=       [self.eos_token_id]





'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_torch_available,
)


lowerCAmelCase    :     int					=						{
    """configuration_trocr""": ["""TROCR_PRETRAINED_CONFIG_ARCHIVE_MAP""", """TrOCRConfig"""],
    """processing_trocr""": ["""TrOCRProcessor"""],
}


try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCAmelCase    :     Optional[Any]					=						[
           """TROCR_PRETRAINED_MODEL_ARCHIVE_LIST""",
           """TrOCRForCausalLM""",
           """TrOCRPreTrainedModel""",
       ]


if TYPE_CHECKING:
       from .configuration_trocr import TROCR_PRETRAINED_CONFIG_ARCHIVE_MAP, TrOCRConfig
       from .processing_trocr import TrOCRProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_trocr import TROCR_PRETRAINED_MODEL_ARCHIVE_LIST, TrOCRForCausalLM, TrOCRPreTrainedModel

else:
       import sys

       lowerCAmelCase    :     Any					=						_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

