


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


snake_case							: Union[str, Any]      =   logging.get_logger(__name__)

snake_case							: Dict      =   '''▁'''

snake_case							: Optional[Any]      =   {'''vocab_file''': '''sentencepiece.bpe.model'''}

snake_case							: Union[str, Any]      =   {
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

snake_case							: Optional[Any]      =   {
    '''facebook/mbart-large-50-one-to-many-mmt''': 10_24,
}

# fmt: off
snake_case							: str      =   ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']







class  snake_case_     (__lowercase   ):
					UpperCAmelCase__      :					str         =    VOCAB_FILES_NAMES
					UpperCAmelCase__      :					Union[str, Any]         =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					UpperCAmelCase__      :					Optional[Any]         =    PRETRAINED_VOCAB_FILES_MAP
					UpperCAmelCase__      :					Union[str, Any]         =    ['''input_ids''', '''attention_mask''']

					UpperCAmelCase__      :					Dict         =    []
					UpperCAmelCase__      :					Optional[Any]         =    []
					def __init__(   self     :Tuple    ,__snake_case     :str    ,__snake_case     :Any=None    ,__snake_case     :List[Any]=None    ,__snake_case     :List[str]="</s>"    ,__snake_case     :Optional[int]="</s>"    ,__snake_case     :List[Any]="<s>"    ,__snake_case     :int="<unk>"    ,__snake_case     :List[Any]="<pad>"    ,__snake_case     :List[str]="<mask>"    ,__snake_case     :Optional[Dict[str, Any]] = None    ,**__snake_case     :List[Any]    ,)      ->					Tuple:
												# Mask token behave like a normal word, i.e. include the space before it
												a__    =  AddedToken(_A    ,lstrip=_A    ,rstrip=_A   ) if isinstance(_A    ,_A   ) else mask_token

												a__    =  {} if sp_model_kwargs is None else sp_model_kwargs

												a__    =  kwargs.get('additional_special_tokens'    ,[]   )
												kwargs["additional_special_tokens"] += [
												    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
												]

												super().__init__(
												    src_lang=_A    ,tgt_lang=_A    ,eos_token=_A    ,unk_token=_A    ,sep_token=_A    ,cls_token=_A    ,pad_token=_A    ,mask_token=_A    ,sp_model_kwargs=self.sp_model_kwargs    ,**_A    ,)

												a__    =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
												self.sp_model.Load(str(_A   )   )
												a__    =  vocab_file

												# Original fairseq vocab and spm vocab must be "aligned":
												# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
												# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
												# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
												# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

												# Mimic fairseq token-to-id alignment for the first 4 token
												a__    =  {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

												# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
												a__    =  1

												a__    =  len(self.sp_model   )
												a__    =  {
												    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_A   )
												}
												a__    =  {v: k for k, v in self.lang_code_to_id.items()}
												a__    =  len(self.sp_model   ) + len(self.lang_code_to_id   ) + self.fairseq_offset

												self.fairseq_tokens_to_ids.update(self.lang_code_to_id   )
												a__    =  {v: k for k, v in self.fairseq_tokens_to_ids.items()}

												a__    =  src_lang if src_lang is not None else 'en_XX'
												a__    =  self.lang_code_to_id[self._src_lang]
												a__    =  tgt_lang
												self.set_src_lang_special_tokens(self._src_lang   )
					@property
					def 				lowerCamelCase__(   self     :List[str]   )      ->					Any:
												return len(self.sp_model   ) + len(self.lang_code_to_id   ) + self.fairseq_offset + 1  # Plus 1 for the mask token
					@property
					def 				lowerCamelCase__(   self     :Optional[int]   )      ->					str:
												return self._src_lang
					@src_lang.setter
					def 				lowerCamelCase__(   self     :Dict    ,__snake_case     :str   )      ->					List[Any]:
												a__    =  new_src_lang
												self.set_src_lang_special_tokens(self._src_lang   )
					def __getstate__(   self     :List[str]   )      ->					int:
												a__    =  self.__dict__.copy()
												a__    =  None
												return state
					def __setstate__(   self     :List[Any]    ,__snake_case     :Dict   )      ->					Optional[int]:
												a__    =  d

												# for backward compatibility
												if not hasattr(self    ,'sp_model_kwargs'   ):
																			a__    =  {}

												a__    =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
												self.sp_model.Load(self.vocab_file   )
					def 				lowerCamelCase__(   self     :Any   )      ->					str:
												a__    =  {self.convert_ids_to_tokens(_A   ): i for i in range(self.vocab_size   )}
												vocab.update(self.added_tokens_encoder   )
												return vocab
					def 				lowerCamelCase__(   self     :str    ,__snake_case     :str   )      ->					Tuple:
												return self.sp_model.encode(_A    ,out_type=_A   )
					def 				lowerCamelCase__(   self     :Dict    ,__snake_case     :str   )      ->					Any:
												if token in self.fairseq_tokens_to_ids:
																			return self.fairseq_tokens_to_ids[token]
												a__    =  self.sp_model.PieceToId(_A   )

												# Need to return unknown token if the SP model returned 0
												return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
					def 				lowerCamelCase__(   self     :int    ,__snake_case     :int   )      ->					Any:
												if index in self.fairseq_ids_to_tokens:
																			return self.fairseq_ids_to_tokens[index]
												return self.sp_model.IdToPiece(index - self.fairseq_offset   )
					def 				lowerCamelCase__(   self     :List[Any]    ,__snake_case     :str   )      ->					Tuple:
												a__    =  []
												a__    =  ''
												a__    =  False
												for token in tokens:
																			# make sure that special tokens are not decoded using sentencepiece model
																			if token in self.all_special_tokens:
																										if not prev_is_special:
																																	out_string += " "
																										out_string += self.sp_model.decode(_A   ) + token
																										a__    =  True
																										a__    =  []
																			else:
																										current_sub_tokens.append(_A   )
																										a__    =  False
												out_string += self.sp_model.decode(_A   )
												return out_string.strip()
					def 				lowerCamelCase__(   self     :Optional[int]    ,__snake_case     :str    ,__snake_case     :Optional[str] = None   )      ->					Any:
												if not os.path.isdir(_A   ):
																			logger.error(F'Vocabulary path ({save_directory}) should be a directory'   )
																			return
												a__    =  os.path.join(
												    _A    ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

												if os.path.abspath(self.vocab_file   ) != os.path.abspath(_A   ) and os.path.isfile(self.vocab_file   ):
																			copyfile(self.vocab_file    ,_A   )
												elif not os.path.isfile(self.vocab_file   ):
																			with open(_A    ,'wb'   ) as fi:
																										a__    =  self.sp_model.serialized_model_proto()
																										fi.write(_A   )

												return (out_vocab_file,)
					def 				lowerCamelCase__(   self     :Dict    ,__snake_case     :List[int]    ,__snake_case     :Optional[List[int]] = None    ,__snake_case     :bool = False   )      ->					Union[str, Any]:

												if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=_A    ,token_ids_a=_A    ,already_has_special_tokens=_A   )

												a__    =  [1] * len(self.prefix_tokens   )
												a__    =  [1] * len(self.suffix_tokens   )
												if token_ids_a is None:
																			return prefix_ones + ([0] * len(_A   )) + suffix_ones
												return prefix_ones + ([0] * len(_A   )) + ([0] * len(_A   )) + suffix_ones
					def 				lowerCamelCase__(   self     :Tuple    ,__snake_case     :List[int]    ,__snake_case     :Optional[List[int]] = None   )      ->					List[str]:
												if token_ids_a is None:
																			return self.prefix_tokens + token_ids_a + self.suffix_tokens
												# We don't expect to process pairs, but leave the pair logic for API consistency
												return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
					def 				lowerCamelCase__(   self     :str    ,__snake_case     :Optional[int]    ,__snake_case     :str    ,__snake_case     :Optional[str]    ,__snake_case     :Optional[str]    ,**__snake_case     :List[Any]   )      ->					Union[str, Any]:
												if src_lang is None or tgt_lang is None:
																			raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'   )
												a__    =  src_lang
												a__    =  self(_A    ,add_special_tokens=_A    ,return_tensors=_A    ,**_A   )
												a__    =  self.convert_tokens_to_ids(_A   )
												a__    =  tgt_lang_id
												return inputs
					def 				lowerCamelCase__(   self     :int    ,__snake_case     :List[str]    ,__snake_case     :str = "en_XX"    ,__snake_case     :Optional[List[str]] = None    ,__snake_case     :str = "ro_RO"    ,**__snake_case     :List[str]    ,)      ->					Optional[int]:
												a__    =  src_lang
												a__    =  tgt_lang
												return super().prepare_seqaseq_batch(_A    ,_A    ,**_A   )
					def 				lowerCamelCase__(   self     :str   )      ->					Optional[int]:
												return self.set_src_lang_special_tokens(self.src_lang   )
					def 				lowerCamelCase__(   self     :List[Any]   )      ->					Optional[Any]:
												return self.set_tgt_lang_special_tokens(self.tgt_lang   )
					def 				lowerCamelCase__(   self     :Tuple    ,__snake_case     :str   )      ->					str:
												a__    =  self.lang_code_to_id[src_lang]
												a__    =  [self.cur_lang_code_id]
												a__    =  [self.eos_token_id]



					def 				lowerCamelCase__(   self     :List[str]    ,__snake_case     :str   )      ->					Optional[int]:
												a__    =  self.lang_code_to_id[tgt_lang]
												a__    =  [self.cur_lang_code_id]
												a__    =  [self.eos_token_id]








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


snake_case							: Any      =   {
    '''configuration_mobilevit''': ['''MOBILEVIT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''MobileViTConfig''', '''MobileViTOnnxConfig'''],
}

try:
						if not is_vision_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						snake_case							: Union[str, Any]      =   ['''MobileViTFeatureExtractor''']
						snake_case							: int      =   ['''MobileViTImageProcessor''']

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						snake_case							: Dict      =   [
						    '''MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''MobileViTForImageClassification''',
						    '''MobileViTForSemanticSegmentation''',
						    '''MobileViTModel''',
						    '''MobileViTPreTrainedModel''',
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						snake_case							: Tuple      =   [
						    '''TF_MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''TFMobileViTForImageClassification''',
						    '''TFMobileViTForSemanticSegmentation''',
						    '''TFMobileViTModel''',
						    '''TFMobileViTPreTrainedModel''',
						]

if TYPE_CHECKING:
						from .configuration_mobilevit import MOBILEVIT_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileViTConfig, MobileViTOnnxConfig

						try:
												if not is_vision_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .feature_extraction_mobilevit import MobileViTFeatureExtractor
												from .image_processing_mobilevit import MobileViTImageProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_mobilevit import (
												    MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    MobileViTForImageClassification,
												    MobileViTForSemanticSegmentation,
												    MobileViTModel,
												    MobileViTPreTrainedModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_mobilevit import (
												    TF_MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFMobileViTForImageClassification,
												    TFMobileViTForSemanticSegmentation,
												    TFMobileViTModel,
												    TFMobileViTPreTrainedModel,
												)


else:
						import sys

						snake_case							: Optional[int]      =   _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




