


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_UpperCAmelCase			:  List[Any]     =logging.get_logger(__name__)

_UpperCAmelCase			:  List[str]     ={"""vocab_file""": """spiece.model"""}

_UpperCAmelCase			:  List[Any]     ={
    """vocab_file""": {
        """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model""",
    }
}







class 				snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						def __init__(				self   ,						__lowercase   ,						__lowercase=False   ,						__lowercase=True   ,						__lowercase=False   ,						__lowercase="<s>"   ,						__lowercase="</s>"   ,						__lowercase="<unk>"   ,						__lowercase="<sep>"   ,						__lowercase="<pad>"   ,						__lowercase="<cls>"   ,						__lowercase="<mask>"   ,						__lowercase=["<eop>", "<eod>"]   ,						__lowercase = None   ,						**__lowercase   ,						)					->    None:
										lowerCAmelCase_ :      str    				=				AddedToken(__lowercase   ,						lstrip=__lowercase   ,						rstrip=__lowercase							) if isinstance(__lowercase   ,						__lowercase							) else mask_token

										lowerCAmelCase_ :      List[Any]    				=				{} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=__lowercase   ,						remove_space=__lowercase   ,						keep_accents=__lowercase   ,						bos_token=__lowercase   ,						eos_token=__lowercase   ,						unk_token=__lowercase   ,						sep_token=__lowercase   ,						pad_token=__lowercase   ,						cls_token=__lowercase   ,						mask_token=__lowercase   ,						additional_special_tokens=__lowercase   ,						sp_model_kwargs=self.sp_model_kwargs   ,						**__lowercase   ,						)

										lowerCAmelCase_ :      str    				=				3

										lowerCAmelCase_ :      Tuple    				=				do_lower_case
										lowerCAmelCase_ :      Any    				=				remove_space
										lowerCAmelCase_ :      Optional[int]    				=				keep_accents
										lowerCAmelCase_ :      Any    				=				vocab_file

										lowerCAmelCase_ :      Optional[int]    				=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
										self.sp_model.Load(__lowercase							)

										try:
														import jieba
										except ModuleNotFoundError as error:
														raise error.__class__(
														    '''You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '''
														    '''See https://pypi.org/project/jieba/ for installation.'''							)
										lowerCAmelCase_ :      Optional[int]    				=				jieba
										lowerCAmelCase_ :      Any    				=				str.maketrans(''' \n'''   ,						'''\u2582\u2583'''							)




						@property
						# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
						def 						lowercase_ (				self							)					->    List[Any]:
										return len(self.sp_model							)




						def 						lowercase_ (				self							)					->    Union[str, Any]:
										lowerCAmelCase_ :      Tuple    				=				{self.convert_ids_to_tokens(__lowercase							): i for i in range(self.vocab_size							)}
										vocab.update(self.added_tokens_encoder							)
										return vocab




						def __getstate__(				self							)					->    List[str]:
										lowerCAmelCase_ :      Union[str, Any]    				=				self.__dict__.copy()
										lowerCAmelCase_ :      List[str]    				=				None
										return state




						def __setstate__(				self   ,						__lowercase							)					->    Tuple:
										lowerCAmelCase_ :      Union[str, Any]    				=				d

										# for backward compatibility
										if not hasattr(self   ,						'''sp_model_kwargs'''							):
														lowerCAmelCase_ :      List[str]    				=				{}

										lowerCAmelCase_ :      str    				=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
										self.sp_model.Load(self.vocab_file							)




						def 						lowercase_ (				self   ,						__lowercase							)					->    Optional[int]:
										if self.remove_space:
														lowerCAmelCase_ :      str    				=				''' '''.join(inputs.strip().split()							)
										else:
														lowerCAmelCase_ :      Optional[Any]    				=				inputs
										lowerCAmelCase_ :      Dict    				=				outputs.replace('''``'''   ,						'''"'''							).replace('''\'\''''   ,						'''"'''							)

										if not self.keep_accents:
														lowerCAmelCase_ :      List[Any]    				=				unicodedata.normalize('''NFKD'''   ,						__lowercase							)
														lowerCAmelCase_ :      List[Any]    				=				''''''.join([c for c in outputs if not unicodedata.combining(__lowercase							)]							)
										if self.do_lower_case:
														lowerCAmelCase_ :      Dict    				=				outputs.lower()

										return outputs




						def 						lowercase_ (				self   ,						__lowercase							)					->    List[str]:
										lowerCAmelCase_ :      List[str]    				=				self.preprocess_text(__lowercase							)
										lowerCAmelCase_ :      Optional[Any]    				=				self.sp_model.encode(__lowercase   ,						out_type=__lowercase							)
										lowerCAmelCase_ :      List[str]    				=				[]
										for piece in pieces:
														if len(__lowercase							) > 1 and piece[-1] == str(''','''							) and piece[-2].isdigit():
																		lowerCAmelCase_ :      Dict    				=				self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowercase   ,						''''''							)							)
																		if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																						if len(cur_pieces[0]							) == 1:
																										lowerCAmelCase_ :      Optional[int]    				=				cur_pieces[1:]
																						else:
																										lowerCAmelCase_ :      List[str]    				=				cur_pieces[0][1:]
																		cur_pieces.append(piece[-1]							)
																		new_pieces.extend(__lowercase							)
														else:
																		new_pieces.append(__lowercase							)

										return new_pieces




						def 						lowercase_ (				self   ,						__lowercase							)					->    int:
										return self.sp_model.PieceToId(__lowercase							)




						def 						lowercase_ (				self   ,						__lowercase							)					->    int:
										return self.sp_model.IdToPiece(__lowercase							)




						def 						lowercase_ (				self   ,						__lowercase							)					->    Tuple:
										lowerCAmelCase_ :      Any    				=				''''''.join(__lowercase							).replace(__lowercase   ,						''' '''							).strip()
										return out_string




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    List[int]:
										lowerCAmelCase_ :      List[Any]    				=				[self.sep_token_id]
										lowerCAmelCase_ :      str    				=				[self.cls_token_id]
										if token_ids_a is None:
														return token_ids_a + sep + cls
										return token_ids_a + sep + token_ids_a + sep + cls




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None   ,						__lowercase = False							)					->    List[int]:

										if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=__lowercase   ,						token_ids_a=__lowercase   ,						already_has_special_tokens=__lowercase							)

										if token_ids_a is not None:
														return ([0] * len(__lowercase							)) + [1] + ([0] * len(__lowercase							)) + [1, 1]
										return ([0] * len(__lowercase							)) + [1, 1]




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    List[int]:
										lowerCAmelCase_ :      Optional[int]    				=				[self.sep_token_id]
										lowerCAmelCase_ :      str    				=				[2]

										if token_ids_a is None:
														return len(token_ids_a + sep							) * [0] + cls_segment_id
										return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    Tuple[str]:
										if not os.path.isdir(__lowercase							):
														logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
														return
										lowerCAmelCase_ :      Any    				=				os.path.join(
										    __lowercase   ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

										if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowercase							) and os.path.isfile(self.vocab_file							):
														copyfile(self.vocab_file   ,						__lowercase							)
										elif not os.path.isfile(self.vocab_file							):
														with open(__lowercase   ,						'''wb'''							) as fi:
																		lowerCAmelCase_ :      Union[str, Any]    				=				self.sp_model.serialized_model_proto()
																		fi.write(__lowercase							)

										return (out_vocab_file,)




						def 						lowercase_ (				self   ,						*__lowercase   ,						**__lowercase							)					->    Optional[Any]:
										lowerCAmelCase_ :      Tuple    				=				super()._decode(*__lowercase   ,						**__lowercase							)
										lowerCAmelCase_ :      Any    				=				text.replace(''' '''   ,						''''''							).replace('''\u2582'''   ,						''' '''							).replace('''\u2583'''   ,						'''\n'''							)
										return text



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_xlnet import XLNetTokenizer
else:
						_UpperCAmelCase			:  Dict     =None


_UpperCAmelCase			:  Tuple     =logging.get_logger(__name__)

_UpperCAmelCase			:  Any     ={"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

_UpperCAmelCase			:  Any     ={
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json""",
    },
}

_UpperCAmelCase			:  Dict     ={
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

_UpperCAmelCase			:  Tuple     ="""▁"""

# Segments (not really needed)
_UpperCAmelCase			:  str     =0
_UpperCAmelCase			:  List[str]     =1
_UpperCAmelCase			:  int     =2
_UpperCAmelCase			:  Any     =3
_UpperCAmelCase			:  List[Any]     =4







class 				snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__      :  int					=							VOCAB_FILES_NAMES
						SCREAMING_SNAKE_CASE__      :  Dict					=							PRETRAINED_VOCAB_FILES_MAP
						SCREAMING_SNAKE_CASE__      :  Optional[Any]					=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						SCREAMING_SNAKE_CASE__      :  Any					=							"""left"""
						SCREAMING_SNAKE_CASE__      :  List[Any]					=							XLNetTokenizer




						def __init__(				self   ,						__lowercase=None   ,						__lowercase=None   ,						__lowercase=False   ,						__lowercase=True   ,						__lowercase=False   ,						__lowercase="<s>"   ,						__lowercase="</s>"   ,						__lowercase="<unk>"   ,						__lowercase="<sep>"   ,						__lowercase="<pad>"   ,						__lowercase="<cls>"   ,						__lowercase="<mask>"   ,						__lowercase=["<eop>", "<eod>"]   ,						**__lowercase   ,						)					->    List[Any]:
										# Mask token behave like a normal word, i.e. include the space before it
										lowerCAmelCase_ :      Any    				=				AddedToken(__lowercase   ,						lstrip=__lowercase   ,						rstrip=__lowercase							) if isinstance(__lowercase   ,						__lowercase							) else mask_token

										super().__init__(
										    vocab_file=__lowercase   ,						tokenizer_file=__lowercase   ,						do_lower_case=__lowercase   ,						remove_space=__lowercase   ,						keep_accents=__lowercase   ,						bos_token=__lowercase   ,						eos_token=__lowercase   ,						unk_token=__lowercase   ,						sep_token=__lowercase   ,						pad_token=__lowercase   ,						cls_token=__lowercase   ,						mask_token=__lowercase   ,						additional_special_tokens=__lowercase   ,						**__lowercase   ,						)

										lowerCAmelCase_ :      List[Any]    				=				3
										lowerCAmelCase_ :      Dict    				=				do_lower_case
										lowerCAmelCase_ :      Dict    				=				remove_space
										lowerCAmelCase_ :      List[str]    				=				keep_accents
										lowerCAmelCase_ :      List[str]    				=				vocab_file
										lowerCAmelCase_ :      str    				=				False if not self.vocab_file else True




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    List[int]:
										lowerCAmelCase_ :      Tuple    				=				[self.sep_token_id]
										lowerCAmelCase_ :      Any    				=				[self.cls_token_id]
										if token_ids_a is None:
														return token_ids_a + sep + cls
										return token_ids_a + sep + token_ids_a + sep + cls




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    List[int]:
										lowerCAmelCase_ :      Optional[Any]    				=				[self.sep_token_id]
										lowerCAmelCase_ :      List[Any]    				=				[2]

										if token_ids_a is None:
														return len(token_ids_a + sep							) * [0] + cls_segment_id
										return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    Tuple[str]:
										if not self.can_save_slow_tokenizer:
														raise ValueError(
														    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
														    '''tokenizer.'''							)

										if not os.path.isdir(__lowercase							):
														logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
														return
										lowerCAmelCase_ :      str    				=				os.path.join(
										    __lowercase   ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

										if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowercase							):
														copyfile(self.vocab_file   ,						__lowercase							)

										return (out_vocab_file,)