





'''simple docstring'''







import inspect
import os
import re

from transformers.configuration_utils import PretrainedConfig
from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
snake_case_     :       Optional[int]						=  "src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
snake_case_     :       Optional[int]						=  direct_transformers_import(PATH_TO_TRANSFORMERS)

snake_case_     :       Optional[Any]						=  transformers.models.auto.configuration_auto.CONFIG_MAPPING

snake_case_     :       int						=  {
    # used to compute the property `self.chunk_length`
    "EncodecConfig": ["overlap"],
    # used as `self.bert_model = BertModel(config, ...)`
    "DPRConfig": True,
    # not used in modeling files, but it's an important information
    "FSMTConfig": ["langs"],
    # used internally in the configuration class file
    "GPTNeoConfig": ["attention_types"],
    # used internally in the configuration class file
    "EsmConfig": ["is_folding_model"],
    # used during training (despite we don't have training script for these models yet)
    "Mask2FormerConfig": ["ignore_value"],
    # `ignore_value` used during training (despite we don't have training script for these models yet)
    # `norm` used in conversion script (despite not using in the modeling file)
    "OneFormerConfig": ["ignore_value", "norm"],
    # used during preprocessing and collation, see `collating_graphormer.py`
    "GraphormerConfig": ["spatial_pos_max"],
    # used internally in the configuration class file
    "T5Config": ["feed_forward_proj"],
    # used internally in the configuration class file
    # `tokenizer_class` get default value `T5Tokenizer` intentionally
    "MT5Config": ["feed_forward_proj", "tokenizer_class"],
    "UMT5Config": ["feed_forward_proj", "tokenizer_class"],
    # used internally in the configuration class file
    "LongT5Config": ["feed_forward_proj"],
    # used internally in the configuration class file
    "SwitchTransformersConfig": ["feed_forward_proj"],
    # having default values other than `1e-5` - we can't fix them without breaking
    "BioGptConfig": ["layer_norm_eps"],
    # having default values other than `1e-5` - we can't fix them without breaking
    "GLPNConfig": ["layer_norm_eps"],
    # having default values other than `1e-5` - we can't fix them without breaking
    "SegformerConfig": ["layer_norm_eps"],
    # having default values other than `1e-5` - we can't fix them without breaking
    "CvtConfig": ["layer_norm_eps"],
    # having default values other than `1e-5` - we can't fix them without breaking
    "PerceiverConfig": ["layer_norm_eps"],
    # used internally to calculate the feature size
    "InformerConfig": ["num_static_real_features", "num_time_features"],
    # used internally to calculate the feature size
    "TimeSeriesTransformerConfig": ["num_static_real_features", "num_time_features"],
    # used internally to calculate the feature size
    "AutoformerConfig": ["num_static_real_features", "num_time_features"],
    # used internally to calculate `mlp_dim`
    "SamVisionConfig": ["mlp_ratio"],
    # For (head) training, but so far not implemented
    "ClapAudioConfig": ["num_classes"],
    # Not used, but providing useful information to users
    "SpeechT5HifiGanConfig": ["sampling_rate"],
}


# TODO (ydshieh): Check the failing cases, try to fix them or move some cases to the above block once we are sure
SPECIAL_CASES_TO_ALLOW.update(
    {
        "CLIPSegConfig": True,
        "DeformableDetrConfig": True,
        "DetaConfig": True,
        "DinatConfig": True,
        "DonutSwinConfig": True,
        "EfficientFormerConfig": True,
        "FSMTConfig": True,
        "JukeboxConfig": True,
        "LayoutLMv2Config": True,
        "MaskFormerSwinConfig": True,
        "MT5Config": True,
        "NatConfig": True,
        "OneFormerConfig": True,
        "PerceiverConfig": True,
        "RagConfig": True,
        "SpeechT5Config": True,
        "SwinConfig": True,
        "Swin2SRConfig": True,
        "Swinv2Config": True,
        "SwitchTransformersConfig": True,
        "TableTransformerConfig": True,
        "TapasConfig": True,
        "TransfoXLConfig": True,
        "UniSpeechConfig": True,
        "UniSpeechSatConfig": True,
        "WavLMConfig": True,
        "WhisperConfig": True,
        # TODO: @Arthur (for `alignment_head` and `alignment_layer`)
        "JukeboxPriorConfig": True,
        # TODO: @Younes (for `is_decoder`)
        "Pix2StructTextConfig": True,
    }
)





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			Dict,			SCREAMING_SNAKE_CASE__      :			Union[str, Any],			SCREAMING_SNAKE_CASE__      :			List[Any],			SCREAMING_SNAKE_CASE__      :			Union[str, Any]   ) ->	Tuple:
						UpperCAmelCase_			:      Union[str, Any]     						=  False
						for attribute in attributes:
												for modeling_source in source_strings:
																		# check if we can find `config.xxx`, `getattr(config, "xxx", ...)` or `getattr(self.config, "xxx", ...)`
																		if (
																		    F"""config.{attribute}""" in modeling_source
																		    or F"""getattr(config, \"{attribute}\"""" in modeling_source
																		    or F"""getattr(self.config, \"{attribute}\"""" in modeling_source
																		):
																								UpperCAmelCase_			:      int     						=  True
																		# Deal with multi-line cases
																		elif (
																		    re.search(
																		        RF"""getattr[ \t\v\n\r\f]*\([ \t\v\n\r\f]*(self\.)?config,[ \t\v\n\r\f]*\"{attribute}\"""",			SCREAMING_SNAKE_CASE__,			)
																		    is not None
																		):
																								UpperCAmelCase_			:      Any     						=  True
																		# `SequenceSummary` is called with `SequenceSummary(config)`
																		elif attribute in [
																		    "summary_type",
																		    "summary_use_proj",
																		    "summary_activation",
																		    "summary_last_dropout",
																		    "summary_proj_to_labels",
																		    "summary_first_dropout",
																		]:
																								if "SequenceSummary" in modeling_source:
																														UpperCAmelCase_			:      Optional[int]     						=  True
																		if attribute_used:
																								break
												if attribute_used:
																		break

    # common and important attributes, even if they do not always appear in the modeling files
						UpperCAmelCase_			:      Union[str, Any]     						=  [
						    '''bos_index''',
						    '''eos_index''',
						    '''pad_index''',
						    '''unk_index''',
						    '''mask_index''',
						    '''image_size''',
						    '''use_cache''',
						    '''out_features''',
						    '''out_indices''',
						]
						UpperCAmelCase_			:      Any     						=  ['''encoder_no_repeat_ngram_size''']

						# Special cases to be allowed
						UpperCAmelCase_			:      Union[str, Any]     						=  True
						if not attribute_used:
												UpperCAmelCase_			:      List[str]     						=  False
												for attribute in attributes:
																		# Allow if the default value in the configuration class is different from the one in `PretrainedConfig`
																		if attribute in ["is_encoder_decoder"] and default_value is True:
																								UpperCAmelCase_			:      Tuple     						=  True
																		elif attribute in ["tie_word_embeddings"] and default_value is False:
																								UpperCAmelCase_			:      Any     						=  True

																		# Allow cases without checking the default value in the configuration class
																		elif attribute in attributes_to_allow + attributes_used_in_generation:
																								UpperCAmelCase_			:      Optional[int]     						=  True
																		elif attribute.endswith('''_token_id'''   ):
																								UpperCAmelCase_			:      Any     						=  True

																		# configuration class specific cases
																		if not case_allowed:
																								UpperCAmelCase_			:      str     						=  SPECIAL_CASES_TO_ALLOW.get(config_class.__name__,			[]   )
																								UpperCAmelCase_			:      Tuple     						=  allowed_cases is True or attribute in allowed_cases

						return attribute_used or case_allowed





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			int   ) ->	str:
						UpperCAmelCase_			:      Any     						=  dict(inspect.signature(config_class.__init__   ).parameters   )
						UpperCAmelCase_			:      List[Any]     						=  [x for x in list(signature.keys()   ) if x not in ['''self''', '''kwargs''']]
						UpperCAmelCase_			:      str     						=  [signature[param].default for param in parameter_names]

						# If `attribute_map` exists, an attribute can have different names to be used in the modeling files, and as long
						# as one variant is used, the test should pass
						UpperCAmelCase_			:      Tuple     						=  {}
						if len(config_class.attribute_map   ) > 0:
												UpperCAmelCase_			:      Optional[int]     						=  {v: k for k, v in config_class.attribute_map.items()}

						# Get the path to modeling source files
						UpperCAmelCase_			:      Optional[int]     						=  inspect.getsourcefile(SCREAMING_SNAKE_CASE__   )
						UpperCAmelCase_			:      Optional[int]     						=  os.path.dirname(SCREAMING_SNAKE_CASE__   )
						# Let's check against all frameworks: as long as one framework uses an attribute, we are good.
						UpperCAmelCase_			:      Optional[int]     						=  [os.path.join(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   ) for fn in os.listdir(SCREAMING_SNAKE_CASE__   ) if fn.startswith('''modeling_'''   )]

						# Get the source code strings
						UpperCAmelCase_			:      List[str]     						=  []
						for path in modeling_paths:
												if os.path.isfile(SCREAMING_SNAKE_CASE__   ):
																		with open(SCREAMING_SNAKE_CASE__   ) as fp:
																								modeling_sources.append(fp.read()   )

						UpperCAmelCase_			:      Dict     						=  []
						for config_param, default_value in zip(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   ):
												# `attributes` here is all the variant names for `config_param`
												UpperCAmelCase_			:      List[Any]     						=  [config_param]
												# some configuration classes have non-empty `attribute_map`, and both names could be used in the
												# corresponding modeling files. As long as one of them appears, it is fine.
												if config_param in reversed_attribute_map:
																		attributes.append(reversed_attribute_map[config_param]   )

												if not check_attribute_being_used(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   ):
																		unused_attributes.append(attributes[0]   )

						return sorted(SCREAMING_SNAKE_CASE__   )





def        lowerCamelCase_      (  ) ->	List[str]:
						UpperCAmelCase_			:      str     						=  {}
						for _config_class in list(CONFIG_MAPPING.values()   ):
												# Skip deprecated models
												if "models.deprecated" in _config_class.__module__:
																		continue
												# Some config classes are not in `CONFIG_MAPPING` (e.g. `CLIPVisionConfig`, `Blip2VisionConfig`, etc.)
												UpperCAmelCase_			:      Any     						=  [
												    cls
												    for name, cls in inspect.getmembers(
												        inspect.getmodule(_config_class   ),			lambda SCREAMING_SNAKE_CASE__   : inspect.isclass(SCREAMING_SNAKE_CASE__   )
												        and issubclass(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )
												        and inspect.getmodule(SCREAMING_SNAKE_CASE__   ) == inspect.getmodule(_config_class   ),			)
												]
												for config_class in config_classes_in_module:
																		UpperCAmelCase_			:      Optional[Any]     						=  check_config_attributes_being_used(SCREAMING_SNAKE_CASE__   )
																		if len(SCREAMING_SNAKE_CASE__   ) > 0:
																								UpperCAmelCase_			:      Optional[int]     						=  unused_attributes

						if len(SCREAMING_SNAKE_CASE__   ) > 0:
												UpperCAmelCase_			:      Dict     						=  '''The following configuration classes contain unused attributes in the corresponding modeling files:\n'''
												for name, attributes in configs_with_unused_attributes.items():
																		error += F"""{name}: {attributes}\n"""

												raise ValueError(SCREAMING_SNAKE_CASE__   )


if __name__ == "__main__":
					check_config_attributes()












'''simple docstring'''







import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case_     :       str						=  logging.get_logger(__name__)

snake_case_     :       int						=  "▁"

snake_case_     :       str						=  {"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

snake_case_     :       int						=  {
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

snake_case_     :       Optional[Any]						=  {
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

snake_case_     :       Dict						=  {
    "ernie-m-base": 5_14,
    "ernie-m-large": 5_14,
}

snake_case_     :       Any						=  {
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}


class 					__a      (lowerCamelCase     ):
							__a					:   List[str]									=       ["input_ids"]

							__a					:   Union[str, Any]									=       VOCAB_FILES_NAMES
							__a					:   Tuple									=       PRETRAINED_INIT_CONFIGURATION
							__a					:   Dict									=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a					:   Optional[int]									=       PRETRAINED_VOCAB_FILES_MAP
							__a					:   Union[str, Any]									=       RESOURCE_FILES_NAMES

							def __init__(							self							:     Union[str, Any]       ,     __magic_name__							:     Dict       ,     __magic_name__							:     int=None       ,     __magic_name__							:     str=False       ,     __magic_name__							:     int="utf8"       ,     __magic_name__							:     Optional[int]="[UNK]"       ,     __magic_name__							:     Dict="[SEP]"       ,     __magic_name__							:     List[Any]="[PAD]"       ,     __magic_name__							:     str="[CLS]"       ,     __magic_name__							:     Optional[int]="[MASK]"       ,     __magic_name__							:     Optional[Dict[str, Any]] = None       ,     **__magic_name__							:     Union[str, Any]       ,     )					->       None:







													"""simple docstring"""
													# Mask token behave like a normal word, i.e. include the space before it and
													# is included in the raw text, there should be a match in a non-normalized sentence.

													UpperCAmelCase_			:      List[Any]     						=  {} if sp_model_kwargs is None else sp_model_kwargs
													super().__init__(
													    do_lower_case=__magic_name__       ,     unk_token=__magic_name__       ,     sep_token=__magic_name__       ,     pad_token=__magic_name__       ,     cls_token=__magic_name__       ,     mask_token=__magic_name__       ,     vocab_file=__magic_name__       ,     encoding=__magic_name__       ,     sp_model_kwargs=self.sp_model_kwargs       ,     **__magic_name__       ,     )
													UpperCAmelCase_			:      Optional[Any]     						=  do_lower_case
													UpperCAmelCase_			:      List[str]     						=  sentencepiece_model_ckpt
													UpperCAmelCase_			:      Tuple     						=  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
													self.sp_model.Load(__magic_name__   )

													# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
													if vocab_file is not None:
																			UpperCAmelCase_			:      List[Any]     						=  self.load_vocab(filepath=__magic_name__   )
													else:
																			UpperCAmelCase_			:      str     						=  {self.sp_model.id_to_piece(__magic_name__   ): id for id in range(self.sp_model.get_piece_size()   )}
													UpperCAmelCase_			:      int     						=  {v: k for k, v in self.vocab.items()}

							def     UpperCAmelCase__	(							self							:     List[str]       ,     __magic_name__							:     Any   )					->       Any:







													"""simple docstring"""
													if text is None:
																			return None

													UpperCAmelCase_			:      str     						=  self.tokenize(__magic_name__   )
													UpperCAmelCase_						,       UpperCAmelCase_			:      str     						=  '''''', []

													for i, ch in enumerate(__magic_name__   ):
																			if ch in self.SP_CHAR_MAPPING:
																									UpperCAmelCase_			:      Optional[int]     						=  self.SP_CHAR_MAPPING.get(__magic_name__   )
																			else:
																									UpperCAmelCase_			:      Union[str, Any]     						=  unicodedata.normalize('''NFKC'''       ,     __magic_name__   )
																			if self.is_whitespace(__magic_name__   ):
																									continue
																			normalized_text += ch
																			char_mapping.extend([i] * len(__magic_name__   )   )

													UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      Union[str, Any]     						=  normalized_text, [], 0

													if self.do_lower_case:
																			UpperCAmelCase_			:      Optional[int]     						=  text.lower()

													for token in split_tokens:
																			if token[:1] == "▁":
																									UpperCAmelCase_			:      Tuple     						=  token[1:]
																			UpperCAmelCase_			:      int     						=  text[offset:].index(__magic_name__   ) + offset
																			UpperCAmelCase_			:      Optional[int]     						=  start + len(__magic_name__   )

																			token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)   )
																			UpperCAmelCase_			:      int     						=  end
													return token_mapping

							@property
							def     UpperCAmelCase__	(							self							:     Any   )					->       Any:







													"""simple docstring"""
													return len(self.vocab   )

							def     UpperCAmelCase__	(							self							:     List[Any]   )					->       int:







													"""simple docstring"""
													return dict(self.vocab       ,     **self.added_tokens_encoder   )

							def __getstate__(							self							:     str   )					->       Any:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[Any]     						=  self.__dict__.copy()
													UpperCAmelCase_			:      Optional[Any]     						=  None
													return state

							def __setstate__(							self							:     str       ,     __magic_name__							:     Any   )					->       Dict:







													"""simple docstring"""
													UpperCAmelCase_			:      Dict     						=  d

													# for backward compatibility
													if not hasattr(self       ,     '''sp_model_kwargs'''   ):
																			UpperCAmelCase_			:      int     						=  {}

													UpperCAmelCase_			:      List[Any]     						=  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
													self.sp_model.Load(self.sentencepiece_model_ckpt   )

							def     UpperCAmelCase__	(							self							:     Optional[int]       ,     __magic_name__							:     Any   )					->       List[str]:







													"""simple docstring"""
													return "".join((self.SP_CHAR_MAPPING.get(__magic_name__       ,     __magic_name__   ) for c in text)   )

							def     UpperCAmelCase__	(							self							:     List[str]       ,     __magic_name__							:     Tuple       ,     __magic_name__							:     Any=False       ,     __magic_name__							:     List[str]=64       ,     __magic_name__							:     List[str]=0.1   )					->       List[str]:







													"""simple docstring"""

													if self.sp_model_kwargs.get('''enable_sampling'''   ) is True:
																			UpperCAmelCase_			:      Dict     						=  True
													if self.sp_model_kwargs.get('''alpha'''   ) is not None:
																			UpperCAmelCase_			:      Union[str, Any]     						=  self.sp_model_kwargs.get('''alpha'''   )
													if self.sp_model_kwargs.get('''nbest_size'''   ) is not None:
																			UpperCAmelCase_			:      Any     						=  self.sp_model_kwargs.get('''nbest_size'''   )

													if not enable_sampling:
																			UpperCAmelCase_			:      Dict     						=  self.sp_model.EncodeAsPieces(__magic_name__   )
													else:
																			UpperCAmelCase_			:      Dict     						=  self.sp_model.SampleEncodeAsPieces(__magic_name__       ,     __magic_name__       ,     __magic_name__   )
													UpperCAmelCase_			:      List[Any]     						=  []
													for pi, piece in enumerate(__magic_name__   ):
																			if piece == SPIECE_UNDERLINE:
																									if not pieces[pi + 1].startswith(__magic_name__   ) and pi != 0:
																															new_pieces.append(__magic_name__   )
																															continue
																									else:
																															continue
																			UpperCAmelCase_			:      List[str]     						=  0
																			for i, chunk in enumerate(__magic_name__   ):
																									if chunk == SPIECE_UNDERLINE:
																															continue
																									if self.is_ch_char(__magic_name__   ) or self.is_punct(__magic_name__   ):
																															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																					new_pieces.append(piece[lst_i:i]   )
																															new_pieces.append(__magic_name__   )
																															UpperCAmelCase_			:      List[Any]     						=  i + 1
																									elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																					new_pieces.append(piece[lst_i:i]   )
																															UpperCAmelCase_			:      List[str]     						=  i
																									elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																					new_pieces.append(piece[lst_i:i]   )
																															UpperCAmelCase_			:      str     						=  i
																			if len(__magic_name__   ) > lst_i:
																									new_pieces.append(piece[lst_i:]   )
													return new_pieces

							def     UpperCAmelCase__	(							self							:     List[Any]       ,     __magic_name__							:     Optional[int]   )					->       List[str]:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[Any]     						=  ''''''.join(__magic_name__   ).replace(__magic_name__       ,     ''' '''   ).strip()
													return out_string

							def     UpperCAmelCase__	(							self							:     List[Any]       ,     __magic_name__							:     Union[str, Any]   )					->       List[str]:







													"""simple docstring"""
													UpperCAmelCase_			:      str     						=  self.convert_ids_to_tokens(__magic_name__   )
													UpperCAmelCase_			:      Optional[Any]     						=  ''''''.join(__magic_name__   ).replace(__magic_name__       ,     ''' '''   ).strip()
													return out_string

							def     UpperCAmelCase__	(							self							:     str       ,     __magic_name__							:     Optional[Any]   )					->       List[Any]:







													"""simple docstring"""
													return self.vocab.get(__magic_name__       ,     self.vocab.get(self.unk_token   )   )

							def     UpperCAmelCase__	(							self							:     Tuple       ,     __magic_name__							:     Union[str, Any]   )					->       Union[str, Any]:







													"""simple docstring"""
													return self.reverse_vocab.get(__magic_name__       ,     self.unk_token   )

							def     UpperCAmelCase__	(							self							:     Tuple       ,     __magic_name__							:     Any       ,     __magic_name__							:     Union[str, Any]=None   )					->       Any:







													"""simple docstring"""
													if token_ids_a is None:
																			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
													UpperCAmelCase_			:      Union[str, Any]     						=  [self.cls_token_id]
													UpperCAmelCase_			:      List[Any]     						=  [self.sep_token_id]
													return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep

							def     UpperCAmelCase__	(							self							:     Any       ,     __magic_name__							:     Optional[Any]       ,     __magic_name__							:     List[str]=None   )					->       int:







													"""simple docstring"""
													if offset_mapping_a is None:
																			return [(0, 0)] + offset_mapping_a + [(0, 0)]

													return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]

							def     UpperCAmelCase__	(							self							:     Dict       ,     __magic_name__							:     Optional[Any]       ,     __magic_name__							:     List[str]=None       ,     __magic_name__							:     Optional[Any]=False   )					->       Optional[int]:







													"""simple docstring"""

													if already_has_special_tokens:
																			if token_ids_a is not None:
																									raise ValueError(
																									    '''You should not supply a second sequence if the provided sequence of '''
																									    '''ids is already formatted with special tokens for the model.'''   )
																			return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

													if token_ids_a is not None:
																			return [1] + ([0] * len(__magic_name__   )) + [1, 1] + ([0] * len(__magic_name__   )) + [1]
													return [1] + ([0] * len(__magic_name__   )) + [1]

							def     UpperCAmelCase__	(							self							:     List[str]       ,     __magic_name__							:     List[int]       ,     __magic_name__							:     Optional[List[int]] = None   )					->       List[int]:







													"""simple docstring"""
													# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
													if token_ids_a is None:
																			# [CLS] X [SEP]
																			return (len(__magic_name__   ) + 2) * [0]

													# [CLS] A [SEP] [SEP] B [SEP]
													return [0] * (len(__magic_name__   ) + 1) + [1] * (len(__magic_name__   ) + 3)

							def     UpperCAmelCase__	(							self							:     Dict       ,     __magic_name__							:     str   )					->       Tuple:







													"""simple docstring"""
													if "\u4e00" <= char <= "\u9fff":
																			return True
													return False

							def     UpperCAmelCase__	(							self							:     int       ,     __magic_name__							:     Optional[int]   )					->       str:







													"""simple docstring"""
													if ("a" <= char <= "z") or ("A" <= char <= "Z"):
																			return True
													return False

							def     UpperCAmelCase__	(							self							:     int       ,     __magic_name__							:     Optional[Any]   )					->       Dict:







													"""simple docstring"""
													if char in ",;:.?!~，；：。？！《》【】":
																			return True
													return False

							def     UpperCAmelCase__	(							self							:     Tuple       ,     __magic_name__							:     Any   )					->       Union[str, Any]:







													"""simple docstring"""
													if char == " " or char == "\t" or char == "\n" or char == "\r":
																			return True
													if len(__magic_name__   ) == 1:
																			UpperCAmelCase_			:      Optional[Any]     						=  unicodedata.category(__magic_name__   )
																			if cat == "Zs":
																									return True
													return False

							def     UpperCAmelCase__	(							self							:     Union[str, Any]       ,     __magic_name__							:     Tuple   )					->       Any:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[Any]     						=  {}
													with io.open(__magic_name__       ,     '''r'''       ,     encoding='''utf-8'''   ) as f:
																			for index, line in enumerate(__magic_name__   ):
																									UpperCAmelCase_			:      List[Any]     						=  line.rstrip('''\n'''   )
																									UpperCAmelCase_			:      Dict     						=  int(__magic_name__   )

													return token_to_idx



							def     UpperCAmelCase__	(							self							:     Dict       ,     __magic_name__							:     str       ,     __magic_name__							:     Optional[str] = None   )					->       Tuple[str]:







													"""simple docstring"""
													UpperCAmelCase_			:      Union[str, Any]     						=  0
													if os.path.isdir(__magic_name__   ):
																			UpperCAmelCase_			:      Any     						=  os.path.join(
																			    __magic_name__       ,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
													else:
																			UpperCAmelCase_			:      List[str]     						=  (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
													with open(__magic_name__       ,     '''w'''       ,     encoding='''utf-8'''   ) as writer:
																			for token, token_index in sorted(self.vocab.items()       ,     key=lambda __magic_name__   : kv[1]   ):
																									if index != token_index:
																															logger.warning(
																															    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																															    ''' Please check that the vocabulary is not corrupted!'''   )
																															UpperCAmelCase_			:      Dict     						=  token_index
																									writer.write(token + '''\n'''   )
																									index += 1

													UpperCAmelCase_			:      Union[str, Any]     						=  os.path.join(__magic_name__       ,     '''sentencepiece.bpe.model'''   )
													with open(__magic_name__       ,     '''wb'''   ) as fi:
																			UpperCAmelCase_			:      Optional[int]     						=  self.sp_model.serialized_model_proto()
																			fi.write(__magic_name__   )

													return (vocab_file,)





