




import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase       :	str				  =     logging.get_logger(__name__)

lowerCamelCase       :	Tuple				  =     "▁"

lowerCamelCase       :	int				  =     {
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
}

lowerCamelCase       :	Dict				  =     {
    "vocab_file": {
        "facebook/s2t-small-librispeech-asr": (
            "https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json"
        ),
    },
    "spm_file": {
        "facebook/s2t-small-librispeech-asr": (
            "https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model"
        )
    },
}

lowerCamelCase       :	str				  =     {
    "facebook/s2t-small-librispeech-asr": 1_024,
}

lowerCamelCase       :	Optional[Any]				  =     ["pt", "fr", "ru", "nl", "ro", "it", "es", "de"]

lowerCamelCase       :	Tuple				  =     {"mustc": MUSTC_LANGS}







class 						A(		UpperCamelCase     ):



 '''simple docstring'''


 UpperCamelCase		     =		VOCAB_FILES_NAMES
 UpperCamelCase		     =		PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase		     =		MAX_MODEL_INPUT_SIZES
 UpperCamelCase		     =		['''input_ids''', '''attention_mask''']

 UpperCamelCase		     =		[]

 def __init__(       self      :						Tuple		,							A_      :						Optional[int]		,							A_      :						List[Any]		,							A_      :						Tuple="<s>"		,							A_      :						Any="</s>"		,							A_      :						str="<pad>"		,							A_      :						Tuple="<unk>"		,							A_      :						List[Any]=False		,							A_      :						Union[str, Any]=False		,							A_      :						Union[str, Any]=None		,							A_      :						Tuple=None		,							A_      :						Optional[Dict[str, Any]] = None		,							**A_      :						int		,							)     ->			None:

       """simple docstring"""






       lowerCamelCase_	  =					{} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           bos_token=A_		,							eos_token=A_		,							unk_token=A_		,							pad_token=A_		,							do_upper_case=A_		,							do_lower_case=A_		,							tgt_lang=A_		,							lang_codes=A_		,							sp_model_kwargs=self.sp_model_kwargs		,							**A_		,							)
       lowerCamelCase_	  =					do_upper_case
       lowerCamelCase_	  =					do_lower_case

       lowerCamelCase_	  =					load_json(A_						)
       lowerCamelCase_	  =					{v: k for k, v in self.encoder.items()}
       lowerCamelCase_	  =					spm_file
       lowerCamelCase_	  =					load_spm(A_		,							self.sp_model_kwargs						)

       if lang_codes is not None:
             lowerCamelCase_	  =					lang_codes
             lowerCamelCase_	  =					LANGUAGES[lang_codes]
             lowerCamelCase_	  =					[f"""<lang:{lang}>""" for lang in self.langs]
             lowerCamelCase_	  =					{lang: self.sp_model.PieceToId(f"""<lang:{lang}>"""						) for lang in self.langs}

             lowerCamelCase_	  =					self.lang_tokens
             lowerCamelCase_	  =					tgt_lang if tgt_lang is not None else self.langs[0]

             self.set_tgt_lang_special_tokens(self._tgt_lang						)
       else:
             lowerCamelCase_	  =					{}

 @property
 def        a__				(       self      :						List[Any]						)     ->			int:

       """simple docstring"""






       return len(self.encoder						)

 @property
 def        a__				(       self      :						Union[str, Any]						)     ->			str:

       """simple docstring"""






       return self._tgt_lang

 @tgt_lang.setter
 def        a__				(       self      :						Dict		,							A_      :						List[str]						)     ->			None:

       """simple docstring"""






       lowerCamelCase_	  =					new_tgt_lang
       self.set_tgt_lang_special_tokens(A_						)

 def        a__				(       self      :						Any		,							A_      :						str						)     ->			None:

       """simple docstring"""






       lowerCamelCase_	  =					self.lang_code_to_id[tgt_lang]
       lowerCamelCase_	  =					[lang_code_id]

 def        a__				(       self      :						Any		,							A_      :						str						)     ->			List[str]:

       """simple docstring"""






       return self.sp_model.encode(A_		,							out_type=A_						)

 def        a__				(       self      :						Optional[int]		,							A_      :						Optional[Any]						)     ->			List[Any]:

       """simple docstring"""






       return self.encoder.get(A_		,							self.encoder[self.unk_token]						)

 def        a__				(       self      :						Optional[int]		,							A_      :						int						)     ->			str:

       """simple docstring"""






       return self.decoder.get(A_		,							self.unk_token						)

 def        a__				(       self      :						List[str]		,							A_      :						List[str]						)     ->			str:

       """simple docstring"""






       lowerCamelCase_	  =					[]
       lowerCamelCase_	  =					''
       for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                   lowerCamelCase_	  =					self.sp_model.decode(A_						)
                   out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
                   lowerCamelCase_	  =					[]
             else:
                   current_sub_tokens.append(A_						)
       lowerCamelCase_	  =					self.sp_model.decode(A_						)
       out_string += decoded.upper() if self.do_upper_case else decoded
       return out_string.strip()

 def        a__				(       self      :						List[str]		,							A_      :						List[str]		,							A_      :						Union[str, Any]=None						)     ->			List[int]:

       """simple docstring"""






       if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + [self.eos_token_id]
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]

 def        a__				(       self      :						Dict		,							A_      :						List[int]		,							A_      :						Optional[List[int]] = None		,							A_      :						bool = False						)     ->			List[int]:

       """simple docstring"""






       if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=A_		,							token_ids_a=A_		,							already_has_special_tokens=A_						)

       lowerCamelCase_	  =					[1] * len(self.prefix_tokens						)
       lowerCamelCase_	  =					[1]
       if token_ids_a is None:
             return prefix_ones + ([0] * len(A_						)) + suffix_ones
       return prefix_ones + ([0] * len(A_						)) + ([0] * len(A_						)) + suffix_ones

 def        a__				(       self      :						int						)     ->			Dict:

       """simple docstring"""






       lowerCamelCase_	  =					self.encoder.copy()
       vocab.update(self.added_tokens_encoder						)
       return vocab

 def __getstate__(       self      :						Tuple						)     ->			Dict:

       """simple docstring"""






       lowerCamelCase_	  =					self.__dict__.copy()
       lowerCamelCase_	  =					None
       return state

 def __setstate__(       self      :						Union[str, Any]		,							A_      :						Dict						)     ->			None:

       """simple docstring"""






       lowerCamelCase_	  =					d

       # for backward compatibility
       if not hasattr(self		,							'sp_model_kwargs'						):
             lowerCamelCase_	  =					{}

       lowerCamelCase_	  =					load_spm(self.spm_file		,							self.sp_model_kwargs						)





 def        a__				(       self      :						Dict		,							A_      :						str		,							A_      :						Optional[str] = None						)     ->			Tuple[str]:

       """simple docstring"""






       lowerCamelCase_	  =					Path(A_						)
       assert save_dir.is_dir(), f"""{save_directory} should be a directory"""
       lowerCamelCase_	  =					save_dir / (
           (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
       )
       lowerCamelCase_	  =					save_dir / (
           (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
       )

       save_json(self.encoder		,							A_						)

       if os.path.abspath(self.spm_file						) != os.path.abspath(A_						) and os.path.isfile(self.spm_file						):
             copyfile(self.spm_file		,							A_						)
       elif not os.path.isfile(self.spm_file						):
             with open(A_		,							'wb'						) as fi:
                   lowerCamelCase_	  =					self.sp_model.serialized_model_proto()
                   fi.write(A_						)

       return (str(A_						), str(A_						))

def     _SCREAMING_SNAKE_CASE    (							lowercase			: str			,							lowercase			: Dict[str, Any]  ):


      '''simple docstring'''



      lowerCamelCase_	  =					sentencepiece.SentencePieceProcessor(**lowercase  )
      spm.Load(str(lowercase  )  )
      return spm

def     _SCREAMING_SNAKE_CASE    (							lowercase			: str  ):


      '''simple docstring'''



      with open(lowercase			,							'r'  ) as f:
            return json.load(lowercase  )






def     _SCREAMING_SNAKE_CASE    (							lowercase			: Dict			,							lowercase			: str  ):


      '''simple docstring'''



      with open(lowercase			,							'w'  ) as f:
            json.dump(lowercase			,							lowercase			,							indent=2  )









def     _SCREAMING_SNAKE_CASE    (							):


      '''simple docstring'''



      lowerCamelCase_	  =					0
      for i in range(1			,							10_01  ):
            total += i**i
      return str(lowercase  )[-10:]


if __name__ == "__main__":
 print(solution())



