



'''simple docstring'''


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowercase_      :				Optional[Any]   					=    logging.get_logger(__name__)

lowercase_      :				List[str]   					=    {'''vocab_file''': '''spiece.model'''}

lowercase_      :				Tuple   					=    {
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

lowercase_      :				Optional[int]   					=    {
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
lowercase_      :				Any   					=    0
lowercase_      :				Tuple   					=    1
lowercase_      :				str   					=    2
lowercase_      :				int   					=    3
lowercase_      :				List[str]   					=    4




class      __UpperCamelCase       (_UpperCAmelCase			):
	__A			  =					VOCAB_FILES_NAMES
	__A			  =					PRETRAINED_VOCAB_FILES_MAP
	__A			  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__A			  =					'''left'''







	def __init__(    self ,				_lowerCAmelCase ,				_lowerCAmelCase=False ,				_lowerCAmelCase=True ,				_lowerCAmelCase=False ,				_lowerCAmelCase="<s>" ,				_lowerCAmelCase="</s>" ,				_lowerCAmelCase="<unk>" ,				_lowerCAmelCase="<sep>" ,				_lowerCAmelCase="<pad>" ,				_lowerCAmelCase="<cls>" ,				_lowerCAmelCase="<mask>" ,				_lowerCAmelCase=["<eop>", "<eod>"] ,				_lowerCAmelCase = None ,				**_lowerCAmelCase ,				)    ->    None:
							'''simple docstring'''






							lowercase        =							AddedToken(_lowerCAmelCase ,				lstrip=_lowerCAmelCase ,				rstrip=_lowerCAmelCase						) if isinstance(_lowerCAmelCase ,				_lowerCAmelCase						) else mask_token

							lowercase        =							{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=_lowerCAmelCase ,				remove_space=_lowerCAmelCase ,				keep_accents=_lowerCAmelCase ,				bos_token=_lowerCAmelCase ,				eos_token=_lowerCAmelCase ,				unk_token=_lowerCAmelCase ,				sep_token=_lowerCAmelCase ,				pad_token=_lowerCAmelCase ,				cls_token=_lowerCAmelCase ,				mask_token=_lowerCAmelCase ,				additional_special_tokens=_lowerCAmelCase ,				sp_model_kwargs=self.sp_model_kwargs ,				**_lowerCAmelCase ,				)

							lowercase        =							3

							lowercase        =							do_lower_case
							lowercase        =							remove_space
							lowercase        =							keep_accents
							lowercase        =							vocab_file

							lowercase        =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(_lowerCAmelCase						)







	@property
	def   _a   (    self						)    ->    List[str]:
							'''simple docstring'''






							return len(self.sp_model						)







	def   _a   (    self						)    ->    Optional[int]:
							'''simple docstring'''






							lowercase        =							{self.convert_ids_to_tokens(_lowerCAmelCase						): i for i in range(self.vocab_size						)}
							vocab.update(self.added_tokens_encoder						)
							return vocab







	def __getstate__(    self						)    ->    List[Any]:
							'''simple docstring'''






							lowercase        =							self.__dict__.copy()
							lowercase        =							None
							return state







	def __setstate__(    self ,				_lowerCAmelCase						)    ->    Tuple:
							'''simple docstring'''






							lowercase        =							d

							# for backward compatibility
							if not hasattr(self ,				"""sp_model_kwargs"""						):
													lowercase        =							{}

							lowercase        =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(self.vocab_file						)







	def   _a   (    self ,				_lowerCAmelCase						)    ->    Union[str, Any]:
							'''simple docstring'''






							if self.remove_space:
													lowercase        =							""" """.join(inputs.strip().split()						)
							else:
													lowercase        =							inputs
							lowercase        =							outputs.replace("""``""" ,				"""\""""						).replace("""''""" ,				"""\""""						)

							if not self.keep_accents:
													lowercase        =							unicodedata.normalize("""NFKD""" ,				_lowerCAmelCase						)
													lowercase        =							"""""".join([c for c in outputs if not unicodedata.combining(_lowerCAmelCase						)]						)
							if self.do_lower_case:
													lowercase        =							outputs.lower()

							return outputs







	def   _a   (    self ,				_lowerCAmelCase						)    ->    List[str]:
							'''simple docstring'''






							lowercase        =							self.preprocess_text(_lowerCAmelCase						)
							lowercase        =							self.sp_model.encode(_lowerCAmelCase ,				out_type=_lowerCAmelCase						)
							lowercase        =							[]
							for piece in pieces:
													if len(_lowerCAmelCase						) > 1 and piece[-1] == str(""","""						) and piece[-2].isdigit():
																			lowercase        =							self.sp_model.EncodeAsPieces(piece[:-1].replace(_lowerCAmelCase ,				""""""						)						)
																			if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																									if len(cur_pieces[0]						) == 1:
																															lowercase        =							cur_pieces[1:]
																									else:
																															lowercase        =							cur_pieces[0][1:]
																			cur_pieces.append(piece[-1]						)
																			new_pieces.extend(_lowerCAmelCase						)
													else:
																			new_pieces.append(_lowerCAmelCase						)

							return new_pieces







	def   _a   (    self ,				_lowerCAmelCase						)    ->    Dict:
							'''simple docstring'''






							return self.sp_model.PieceToId(_lowerCAmelCase						)







	def   _a   (    self ,				_lowerCAmelCase						)    ->    Any:
							'''simple docstring'''






							return self.sp_model.IdToPiece(_lowerCAmelCase						)







	def   _a   (    self ,				_lowerCAmelCase						)    ->    str:
							'''simple docstring'''






							lowercase        =							"""""".join(_lowerCAmelCase						).replace(_lowerCAmelCase ,				""" """						).strip()
							return out_string







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = False ,				_lowerCAmelCase = None ,				_lowerCAmelCase = True ,				**_lowerCAmelCase ,				)    ->    str:
							'''simple docstring'''






							lowercase        =							kwargs.pop("""use_source_tokenizer""" ,				_lowerCAmelCase						)

							lowercase        =							self.convert_ids_to_tokens(_lowerCAmelCase ,				skip_special_tokens=_lowerCAmelCase						)

							# To avoid mixing byte-level and unicode for byte-level BPT
							# we need to build string separately for added tokens and byte-level tokens
							# cf. https://github.com/huggingface/transformers/issues/1133
							lowercase        =							[]
							lowercase        =							[]
							for token in filtered_tokens:
													if skip_special_tokens and token in self.all_special_ids:
																			continue
													if token in self.added_tokens_encoder:
																			if current_sub_text:
																									sub_texts.append(self.convert_tokens_to_string(_lowerCAmelCase						)						)
																									lowercase        =							[]
																			sub_texts.append(_lowerCAmelCase						)
													else:
																			current_sub_text.append(_lowerCAmelCase						)
							if current_sub_text:
													sub_texts.append(self.convert_tokens_to_string(_lowerCAmelCase						)						)

							# Mimic the behavior of the Rust tokenizer:
							# By default, there are no spaces between special tokens
							lowercase        =							"""""".join(_lowerCAmelCase						)

							lowercase        =							(
							    clean_up_tokenization_spaces
							    if clean_up_tokenization_spaces is not None
							    else self.clean_up_tokenization_spaces
							)
							if clean_up_tokenization_spaces:
													lowercase        =							self.clean_up_tokenization(_lowerCAmelCase						)
													return clean_text
							else:
													return text







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None						)    ->    List[int]:
							'''simple docstring'''






							lowercase        =							[self.sep_token_id]
							lowercase        =							[self.cls_token_id]
							if token_ids_a is None:
													return token_ids_a + sep + cls
							return token_ids_a + sep + token_ids_a + sep + cls







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None ,				_lowerCAmelCase = False						)    ->    List[int]:
							'''simple docstring'''






							if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=_lowerCAmelCase ,				token_ids_a=_lowerCAmelCase ,				already_has_special_tokens=_lowerCAmelCase						)

							if token_ids_a is not None:
													return ([0] * len(_lowerCAmelCase						)) + [1] + ([0] * len(_lowerCAmelCase						)) + [1, 1]
							return ([0] * len(_lowerCAmelCase						)) + [1, 1]







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None						)    ->    List[int]:
							'''simple docstring'''






							lowercase        =							[self.sep_token_id]
							lowercase        =							[2]

							if token_ids_a is None:
													return len(token_ids_a + sep						) * [0] + cls_segment_id
							return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None						)    ->    Tuple[str]:
							'''simple docstring'''






							if not os.path.isdir(_lowerCAmelCase						):
													logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
													return
							lowercase        =							os.path.join(
							    _lowerCAmelCase ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

							if os.path.abspath(self.vocab_file						) != os.path.abspath(_lowerCAmelCase						) and os.path.isfile(self.vocab_file						):
													copyfile(self.vocab_file ,				_lowerCAmelCase						)
							elif not os.path.isfile(self.vocab_file						):
													with open(_lowerCAmelCase ,				"""wb"""						) as fi:
																			lowercase        =							self.sp_model.serialized_model_proto()
																			fi.write(_lowerCAmelCase						)

							return (out_vocab_file,)







'''simple docstring'''


# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


lowercase_      :				List[str]   					=    '''3'''

print('''Python version:''', sys.version)

print('''OS platform:''', platform.platform())
print('''OS architecture:''', platform.machine())

try:
						import torch

						print('''Torch version:''', torch.__version__)
						print('''Cuda available:''', torch.cuda.is_available())
						print('''Cuda version:''', torch.version.cuda)
						print('''CuDNN version:''', torch.backends.cudnn.version())
						print('''Number of GPUs available:''', torch.cuda.device_count())
except ImportError:
						print('''Torch version:''', None)

try:
						import transformers

						print('''transformers version:''', transformers.__version__)
except ImportError:
						print('''transformers version:''', None)


