




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_xlnet import XLNetTokenizer
else:
		__lowerCAmelCase	:       str     =	None


__lowerCAmelCase	:       Tuple     =	logging.get_logger(__name__)

__lowerCAmelCase	:       Optional[int]     =	{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase	:       List[Any]     =	{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json',
    },
}

__lowerCAmelCase	:       Union[str, Any]     =	{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

__lowerCAmelCase	:       List[str]     =	'▁'

# Segments (not really needed)
__lowerCAmelCase	:       Any     =	0
__lowerCAmelCase	:       Union[str, Any]     =	1
__lowerCAmelCase	:       List[str]     =	2
__lowerCAmelCase	:       Optional[Any]     =	3
__lowerCAmelCase	:       List[str]     =	4




class 					snake_case__							(_UpperCamelCase     ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_ :							Tuple											=      VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE_ :							List[Any]											=      PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE_ :							str											=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE_ :							Union[str, Any]											=      """left"""
				SCREAMING_SNAKE_CASE_ :							Tuple											=      XLNetTokenizer





				def __init__(   self       :  Optional[Any]      ,   __lowerCamelCase       :  int=None      ,   __lowerCamelCase       :  Dict=None      ,   __lowerCamelCase       :  Optional[int]=False      ,   __lowerCamelCase       :  List[str]=True      ,   __lowerCamelCase       :  List[str]=False      ,   __lowerCamelCase       :  Tuple="<s>"      ,   __lowerCamelCase       :  List[Any]="</s>"      ,   __lowerCamelCase       :  str="<unk>"      ,   __lowerCamelCase       :  Any="<sep>"      ,   __lowerCamelCase       :  Optional[int]="<pad>"      ,   __lowerCamelCase       :  List[str]="<cls>"      ,   __lowerCamelCase       :  Optional[int]="<mask>"      ,   __lowerCamelCase       :  Union[str, Any]=["<eop>", "<eod>"]      ,   **__lowerCamelCase       :  Any      ,   )      ->  Optional[Any]:
							# Mask token behave like a normal word, i.e. include the space before it
							a					=    AddedToken(__lowerCamelCase      ,   lstrip=__lowerCamelCase      ,   rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase      ,   __lowerCamelCase							) else mask_token

							super().__init__(
							    vocab_file=__lowerCamelCase      ,   tokenizer_file=__lowerCamelCase      ,   do_lower_case=__lowerCamelCase      ,   remove_space=__lowerCamelCase      ,   keep_accents=__lowerCamelCase      ,   bos_token=__lowerCamelCase      ,   eos_token=__lowerCamelCase      ,   unk_token=__lowerCamelCase      ,   sep_token=__lowerCamelCase      ,   pad_token=__lowerCamelCase      ,   cls_token=__lowerCamelCase      ,   mask_token=__lowerCamelCase      ,   additional_special_tokens=__lowerCamelCase      ,   **__lowerCamelCase      ,   )

							a					=    3
							a					=    do_lower_case
							a					=    remove_space
							a					=    keep_accents
							a					=    vocab_file
							a					=    False if not self.vocab_file else True





				def 							__UpperCAmelCase							(   self       :  Any      ,   __lowerCamelCase       :  List[int]      ,   __lowerCamelCase       :  Optional[List[int]] = None							)      ->  List[int]:
							a					=    [self.sep_token_id]
							a					=    [self.cls_token_id]
							if token_ids_a is None:
										return token_ids_a + sep + cls
							return token_ids_a + sep + token_ids_a + sep + cls





				def 							__UpperCAmelCase							(   self       :  str      ,   __lowerCamelCase       :  List[int]      ,   __lowerCamelCase       :  Optional[List[int]] = None							)      ->  List[int]:
							a					=    [self.sep_token_id]
							a					=    [2]

							if token_ids_a is None:
										return len(token_ids_a + sep							) * [0] + cls_segment_id
							return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id





				def 							__UpperCAmelCase							(   self       :  Optional[Any]      ,   __lowerCamelCase       :  str      ,   __lowerCamelCase       :  Optional[str] = None							)      ->  Tuple[str]:
							if not self.can_save_slow_tokenizer:
										raise ValueError(
										    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
										    "tokenizer."							)

							if not os.path.isdir(__lowerCamelCase							):
										logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
										return
							a					=    os.path.join(
							    __lowerCamelCase      ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

							if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowerCamelCase							):
										copyfile(self.vocab_file      ,   __lowerCamelCase							)

							return (out_vocab_file,)












import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase	:       int     =	logging.get_logger(__name__)


__lowerCAmelCase	:       Optional[int]     =	'▁'

__lowerCAmelCase	:       Union[str, Any]     =	{'vocab_file': 'spiece.model'}

__lowerCAmelCase	:       int     =	{
    'vocab_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'
        )
    }
}

__lowerCAmelCase	:       Any     =	{
    'google/reformer-crime-and-punishment': 52_4288,
}




class 					snake_case__							(_UpperCamelCase     ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_ :							Dict											=      VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE_ :							int											=      PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE_ :							List[Any]											=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE_ :							Optional[int]											=      ["""input_ids""", """attention_mask"""]





				def __init__(   self       :  List[Any]      ,   __lowerCamelCase       :  List[str]      ,   __lowerCamelCase       :  Dict="</s>"      ,   __lowerCamelCase       :  Dict="<unk>"      ,   __lowerCamelCase       :  Dict=[]      ,   __lowerCamelCase       :  Optional[Dict[str, Any]] = None      ,   **__lowerCamelCase       :  Dict      ,   )      ->  None:
							a					=    {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    eos_token=__lowerCamelCase      ,   unk_token=__lowerCamelCase      ,   additional_special_tokens=__lowerCamelCase      ,   sp_model_kwargs=self.sp_model_kwargs      ,   **__lowerCamelCase      ,   )

							a					=    vocab_file
							a					=    spm.SentencePieceProcessor(**self.sp_model_kwargs							)
							self.sp_model.Load(__lowerCamelCase							)





				@property
				def 							__UpperCAmelCase							(   self       :  Optional[int]							)      ->  int:
							return self.sp_model.get_piece_size()





				def 							__UpperCAmelCase							(   self       :  Tuple							)      ->  Dict[str, int]:
							a					=    {self.convert_ids_to_tokens(__lowerCamelCase							): i for i in range(self.vocab_size							)}
							vocab.update(self.added_tokens_encoder							)
							return vocab





				def __getstate__(   self       :  Optional[Any]							)      ->  Optional[Any]:
							a					=    self.__dict__.copy()
							a					=    None
							return state





				def __setstate__(   self       :  str      ,   __lowerCamelCase       :  Tuple							)      ->  List[Any]:
							a					=    d

							# for backward compatibility
							if not hasattr(self      ,   "sp_model_kwargs"							):
										a					=    {}

							a					=    spm.SentencePieceProcessor(**self.sp_model_kwargs							)
							self.sp_model.Load(self.vocab_file							)





				def 							__UpperCAmelCase							(   self       :  int      ,   __lowerCamelCase       :  str							)      ->  List[str]:
							return self.sp_model.encode(__lowerCamelCase      ,   out_type=__lowerCamelCase							)





				def 							__UpperCAmelCase							(   self       :  List[Any]      ,   __lowerCamelCase       :  Dict							)      ->  Any:
							return self.sp_model.piece_to_id(__lowerCamelCase							)





				def 							__UpperCAmelCase							(   self       :  int      ,   __lowerCamelCase       :  Union[str, Any]							)      ->  str:
							if index < self.sp_model.get_piece_size():
										a					=    self.sp_model.IdToPiece(__lowerCamelCase							)
							return token





				def 							__UpperCAmelCase							(   self       :  Optional[int]      ,   __lowerCamelCase       :  Optional[Any]							)      ->  List[Any]:
							a					=    []
							a					=    ""
							for token in tokens:
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
													out_string += self.sp_model.decode(__lowerCamelCase							) + token
													a					=    []
										else:
													current_sub_tokens.append(__lowerCamelCase							)
							out_string += self.sp_model.decode(__lowerCamelCase							)
							return out_string.strip()





				def 							__UpperCAmelCase							(   self       :  Any      ,   __lowerCamelCase       :  str      ,   __lowerCamelCase       :  Optional[str] = None							)      ->  Tuple[str]:
							if not os.path.isdir(__lowerCamelCase							):
										logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
										return
							a					=    os.path.join(
							    __lowerCamelCase      ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

							if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowerCamelCase							) and os.path.isfile(self.vocab_file							):
										copyfile(self.vocab_file      ,   __lowerCamelCase							)
							elif not os.path.isfile(self.vocab_file							):
										with open(__lowerCamelCase      ,   "wb"							) as fi:
													a					=    self.sp_model.serialized_model_proto()
													fi.write(__lowerCamelCase							)

							return (out_vocab_file,)






