



'''simple docstring'''


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowercase_      :				Optional[Any]   					=    logging.get_logger(__name__)

lowercase_      :				Optional[int]   					=    {'''vocab_file''': '''spiece.model'''}

lowercase_      :				Optional[Any]   					=    {
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    }
}




class      __UpperCamelCase       (_UpperCAmelCase			):







 def __init__(    self ,				_lowerCAmelCase ,				_lowerCAmelCase=False ,				_lowerCAmelCase=True ,				_lowerCAmelCase=False ,				_lowerCAmelCase="<s>" ,				_lowerCAmelCase="</s>" ,				_lowerCAmelCase="<unk>" ,				_lowerCAmelCase="<sep>" ,				_lowerCAmelCase="<pad>" ,				_lowerCAmelCase="<cls>" ,				_lowerCAmelCase="<mask>" ,				_lowerCAmelCase=["<eop>", "<eod>"] ,				_lowerCAmelCase = None ,				**_lowerCAmelCase ,				)    ->    None:
       '''simple docstring'''






       lowercase        =							AddedToken(_lowerCAmelCase ,				lstrip=_lowerCAmelCase ,				rstrip=_lowerCAmelCase						) if isinstance(_lowerCAmelCase ,				_lowerCAmelCase						) else mask_token

       lowercase        =							{} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=_lowerCAmelCase ,				remove_space=_lowerCAmelCase ,				keep_accents=_lowerCAmelCase ,				bos_token=_lowerCAmelCase ,				eos_token=_lowerCAmelCase ,				unk_token=_lowerCAmelCase ,				sep_token=_lowerCAmelCase ,				pad_token=_lowerCAmelCase ,				cls_token=_lowerCAmelCase ,				mask_token=_lowerCAmelCase ,				additional_special_tokens=_lowerCAmelCase ,				sp_model_kwargs=self.sp_model_kwargs ,				**_lowerCAmelCase ,				)

       lowercase        =							3

       lowercase        =							do_lower_case
       lowercase        =							remove_space
       lowercase        =							keep_accents
       lowercase        =							vocab_file

       lowercase        =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
       self.sp_model.Load(_lowerCAmelCase						)

       try:
             import jieba
       except ModuleNotFoundError as error:
             raise error.__class__(
                 """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
                 """See https://pypi.org/project/jieba/ for installation."""						)
       lowercase        =							jieba
       lowercase        =							str.maketrans(""" \n""" ,				"""\u2582\u2583"""						)







 @property
 # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
 def   _a   (    self						)    ->    Dict:
       '''simple docstring'''






       return len(self.sp_model						)







 def   _a   (    self						)    ->    Dict:
       '''simple docstring'''






       lowercase        =							{self.convert_ids_to_tokens(_lowerCAmelCase						): i for i in range(self.vocab_size						)}
       vocab.update(self.added_tokens_encoder						)
       return vocab







 def __getstate__(    self						)    ->    str:
       '''simple docstring'''






       lowercase        =							self.__dict__.copy()
       lowercase        =							None
       return state







 def __setstate__(    self ,				_lowerCAmelCase						)    ->    List[Any]:
       '''simple docstring'''






       lowercase        =							d

       # for backward compatibility
       if not hasattr(self ,				"""sp_model_kwargs"""						):
             lowercase        =							{}

       lowercase        =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
       self.sp_model.Load(self.vocab_file						)







 def   _a   (    self ,				_lowerCAmelCase						)    ->    Tuple:
       '''simple docstring'''






       if self.remove_space:
             lowercase        =							""" """.join(inputs.strip().split()						)
       else:
             lowercase        =							inputs
       lowercase        =							outputs.replace("""``""" ,				"""\""""						).replace("""''""" ,				"""\""""						)

       if not self.keep_accents:
             lowercase        =							unicodedata.normalize("""NFKD""" ,				_lowerCAmelCase						)
             lowercase        =							"""""".join([c for c in outputs if not unicodedata.combining(_lowerCAmelCase						)]						)
       if self.do_lower_case:
             lowercase        =							outputs.lower()

       return outputs







 def   _a   (    self ,				_lowerCAmelCase						)    ->    List[str]:
       '''simple docstring'''






       lowercase        =							self.preprocess_text(_lowerCAmelCase						)
       lowercase        =							self.sp_model.encode(_lowerCAmelCase ,				out_type=_lowerCAmelCase						)
       lowercase        =							[]
       for piece in pieces:
             if len(_lowerCAmelCase						) > 1 and piece[-1] == str(""","""						) and piece[-2].isdigit():
                   lowercase        =							self.sp_model.EncodeAsPieces(piece[:-1].replace(_lowerCAmelCase ,				""""""						)						)
                   if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                         if len(cur_pieces[0]						) == 1:
                               lowercase        =							cur_pieces[1:]
                         else:
                               lowercase        =							cur_pieces[0][1:]
                   cur_pieces.append(piece[-1]						)
                   new_pieces.extend(_lowerCAmelCase						)
             else:
                   new_pieces.append(_lowerCAmelCase						)

       return new_pieces







 def   _a   (    self ,				_lowerCAmelCase						)    ->    List[Any]:
       '''simple docstring'''






       return self.sp_model.PieceToId(_lowerCAmelCase						)







 def   _a   (    self ,				_lowerCAmelCase						)    ->    Any:
       '''simple docstring'''






       return self.sp_model.IdToPiece(_lowerCAmelCase						)







 def   _a   (    self ,				_lowerCAmelCase						)    ->    str:
       '''simple docstring'''






       lowercase        =							"""""".join(_lowerCAmelCase						).replace(_lowerCAmelCase ,				""" """						).strip()
       return out_string







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None						)    ->    List[int]:
       '''simple docstring'''






       lowercase        =							[self.sep_token_id]
       lowercase        =							[self.cls_token_id]
       if token_ids_a is None:
             return token_ids_a + sep + cls
       return token_ids_a + sep + token_ids_a + sep + cls







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None ,				_lowerCAmelCase = False						)    ->    List[int]:
       '''simple docstring'''






       if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_lowerCAmelCase ,				token_ids_a=_lowerCAmelCase ,				already_has_special_tokens=_lowerCAmelCase						)

       if token_ids_a is not None:
             return ([0] * len(_lowerCAmelCase						)) + [1] + ([0] * len(_lowerCAmelCase						)) + [1, 1]
       return ([0] * len(_lowerCAmelCase						)) + [1, 1]







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None						)    ->    List[int]:
       '''simple docstring'''






       lowercase        =							[self.sep_token_id]
       lowercase        =							[2]

       if token_ids_a is None:
             return len(token_ids_a + sep						) * [0] + cls_segment_id
       return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None						)    ->    Tuple[str]:
       '''simple docstring'''






       if not os.path.isdir(_lowerCAmelCase						):
             logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
             return
       lowercase        =							os.path.join(
           _lowerCAmelCase ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

       if os.path.abspath(self.vocab_file						) != os.path.abspath(_lowerCAmelCase						) and os.path.isfile(self.vocab_file						):
             copyfile(self.vocab_file ,				_lowerCAmelCase						)
       elif not os.path.isfile(self.vocab_file						):
             with open(_lowerCAmelCase ,				"""wb"""						) as fi:
                   lowercase        =							self.sp_model.serialized_model_proto()
                   fi.write(_lowerCAmelCase						)

       return (out_vocab_file,)







 def   _a   (    self ,				*_lowerCAmelCase ,				**_lowerCAmelCase						)    ->    Optional[Any]:
       '''simple docstring'''






       lowercase        =							super()._decode(*_lowerCAmelCase ,				**_lowerCAmelCase						)
       lowercase        =							text.replace(""" """ ,				""""""						).replace("""\u2582""" ,				""" """						).replace("""\u2583""" ,				"""\n"""						)
       return text







'''simple docstring'''


import os







def      SCREAMING_SNAKE_CASE				(				):
						lowercase        =							os.path.join(os.path.dirname(lowercase_						) ,  """num.txt"""						)
						with open(lowercase_						) as file_hand:
												return str(sum(int(lowercase_						) for line in file_hand						)						)[:10]


if __name__ == "__main__":
						print(solution())


