


import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map


def      a	(							SCREAMING_SNAKE_CASE_      :		Union[dict, list, tuple, torch.Tensor]       ):






							"""simple docstring"""





							UpperCamelCase    :	Tuple   	=		[]
							if isinstance(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       ):
														for v in tree.values():
																					shapes.extend(_fetch_dims(SCREAMING_SNAKE_CASE_       )       )
							elif isinstance(SCREAMING_SNAKE_CASE_							,			(list, tuple)       ):
														for t in tree:
																					shapes.extend(_fetch_dims(SCREAMING_SNAKE_CASE_       )       )
							elif isinstance(SCREAMING_SNAKE_CASE_							,			torch.Tensor       ):
														shapes.append(tree.shape       )
							else:
														raise ValueError('''Not supported'''       )

							return shapes


@torch.jit.ignore
def      a	(							SCREAMING_SNAKE_CASE_      :		int							,			SCREAMING_SNAKE_CASE_      :		Tuple[int, ...]       ):






							"""simple docstring"""





							UpperCamelCase    :	Dict   	=		[]
							for d in reversed(SCREAMING_SNAKE_CASE_       ):
														idx.append(flat_idx % d       )
														UpperCamelCase    :	Optional[Any]   	=		flat_idx // d

							return tuple(reversed(SCREAMING_SNAKE_CASE_       )       )


@torch.jit.ignore
def      a	(							SCREAMING_SNAKE_CASE_      :		Sequence[int]							,			SCREAMING_SNAKE_CASE_      :		Sequence[int]							,			SCREAMING_SNAKE_CASE_      :		Sequence[int]							,			SCREAMING_SNAKE_CASE_      :		Optional[Sequence[bool]] = None							,			SCREAMING_SNAKE_CASE_      :		Optional[Sequence[bool]] = None							,			):






							"""simple docstring"""





							def reduce_edge_list(SCREAMING_SNAKE_CASE_      :		List[bool]       ) -> None:
														UpperCamelCase    :	Optional[int]   	=		True
														for i in range(len(SCREAMING_SNAKE_CASE_       )       ):
																					UpperCamelCase    :	List[Any]   	=		-1 * (i + 1)
																					l[reversed_idx] &= tally
																					UpperCamelCase    :	Optional[int]   	=		l[reversed_idx]

							if start_edges is None:
														UpperCamelCase    :	Any   	=		[s == 0 for s in start]
														reduce_edge_list(SCREAMING_SNAKE_CASE_       )
							if end_edges is None:
														UpperCamelCase    :	Union[str, Any]   	=		[e == (d - 1) for e, d in zip(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       )]
														reduce_edge_list(SCREAMING_SNAKE_CASE_       )

							# Base cases. Either start/end are empty and we're done, or the final,
							# one-dimensional tensor can be simply sliced
							if len(SCREAMING_SNAKE_CASE_       ) == 0:
														return [()]
							elif len(SCREAMING_SNAKE_CASE_       ) == 1:
														return [(slice(start[0]							,			end[0] + 1       ),)]

							UpperCamelCase    :	List[Tuple[slice, ...]]   	=		[]
							UpperCamelCase    :	List[slice]   	=		[]

							# Dimensions common to start and end can be selected directly
							for s, e in zip(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       ):
														if s == e:
																					path_list.append(slice(SCREAMING_SNAKE_CASE_							,			s + 1       )       )
														else:
																					break

							UpperCamelCase    :	Tuple[slice, ...]   	=		tuple(SCREAMING_SNAKE_CASE_       )
							UpperCamelCase    :	Dict   	=		len(SCREAMING_SNAKE_CASE_       )

							# start == end, and we're done
							if divergence_idx == len(SCREAMING_SNAKE_CASE_       ):
														return [path]

							def upper() -> Tuple[Tuple[slice, ...], ...]:
														assert start_edges is not None
														assert end_edges is not None

														UpperCamelCase    :	str   	=		start[divergence_idx]
														return tuple(
														    path + (slice(SCREAMING_SNAKE_CASE_							,			sdi + 1       ),) + s
														    for s in _get_minimal_slice_set(
														        start[divergence_idx + 1 :]							,			[d - 1 for d in dims[divergence_idx + 1 :]]							,			dims[divergence_idx + 1 :]							,			start_edges=start_edges[divergence_idx + 1 :]							,			end_edges=[True for _ in end_edges[divergence_idx + 1 :]]							,			)       )

							def lower() -> Tuple[Tuple[slice, ...], ...]:
														assert start_edges is not None
														assert end_edges is not None

														UpperCamelCase    :	str   	=		end[divergence_idx]
														return tuple(
														    path + (slice(SCREAMING_SNAKE_CASE_							,			edi + 1       ),) + s
														    for s in _get_minimal_slice_set(
														        [0 for _ in start[divergence_idx + 1 :]]							,			end[divergence_idx + 1 :]							,			dims[divergence_idx + 1 :]							,			start_edges=[True for _ in start_edges[divergence_idx + 1 :]]							,			end_edges=end_edges[divergence_idx + 1 :]							,			)       )

							# If both start and end are at the edges of the subtree rooted at
							# divergence_idx, we can just select the whole subtree at once
							if start_edges[divergence_idx] and end_edges[divergence_idx]:
														slices.append(path + (slice(start[divergence_idx]							,			end[divergence_idx] + 1       ),)       )
							# If just start is at the edge, we can grab almost all of the subtree,
							# treating only the ragged bottom edge as an edge case
							elif start_edges[divergence_idx]:
														slices.append(path + (slice(start[divergence_idx]							,			end[divergence_idx]       ),)       )
														slices.extend(lower()       )
							# Analogous to the previous case, but the top is ragged this time
							elif end_edges[divergence_idx]:
														slices.extend(upper()       )
														slices.append(path + (slice(start[divergence_idx] + 1							,			end[divergence_idx] + 1       ),)       )
							# If both sides of the range are ragged, we need to handle both sides
							# separately. If there's contiguous meat in between them, we can index it
							# in one big chunk
							else:
														slices.extend(upper()       )
														UpperCamelCase    :	int   	=		end[divergence_idx] - start[divergence_idx]
														if middle_ground > 1:
																					slices.append(path + (slice(start[divergence_idx] + 1							,			end[divergence_idx]       ),)       )
														slices.extend(lower()       )

							return slices


@torch.jit.ignore
def      a	(							SCREAMING_SNAKE_CASE_      :		torch.Tensor							,			SCREAMING_SNAKE_CASE_      :		int							,			SCREAMING_SNAKE_CASE_      :		int							,			SCREAMING_SNAKE_CASE_      :		int       ):






							"""simple docstring"""





							UpperCamelCase    :	int   	=		t.shape[:no_batch_dims]
							UpperCamelCase    :	Union[str, Any]   	=		list(_flat_idx_to_idx(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       )       )
							# _get_minimal_slice_set is inclusive
							UpperCamelCase    :	Any   	=		list(_flat_idx_to_idx(flat_end - 1							,			SCREAMING_SNAKE_CASE_       )       )

							# Get an ordered list of slices to perform
							UpperCamelCase    :	Optional[int]   	=		_get_minimal_slice_set(
							    SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_							,			)

							UpperCamelCase    :	Tuple   	=		[t[s] for s in slices]

							return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]       ) for s in sliced_tensors]       )


def      a	(							SCREAMING_SNAKE_CASE_      :		Callable							,			SCREAMING_SNAKE_CASE_      :		Dict[str, Any]							,			SCREAMING_SNAKE_CASE_      :		int							,			SCREAMING_SNAKE_CASE_      :		int							,			SCREAMING_SNAKE_CASE_      :		bool = False							,			SCREAMING_SNAKE_CASE_      :		Any = None							,			SCREAMING_SNAKE_CASE_      :		bool = False							,			):






							"""simple docstring"""





							if not (len(SCREAMING_SNAKE_CASE_       ) > 0):
														raise ValueError('''Must provide at least one input'''       )

							UpperCamelCase    :	str   	=		[shape[:no_batch_dims] for shape in _fetch_dims(SCREAMING_SNAKE_CASE_       )]
							UpperCamelCase    :	str   	=		tuple([max(SCREAMING_SNAKE_CASE_       ) for s in zip(*SCREAMING_SNAKE_CASE_       )]       )

							def _prep_inputs(SCREAMING_SNAKE_CASE_      :		torch.Tensor       ) -> torch.Tensor:
														if not low_mem:
																					if not sum(t.shape[:no_batch_dims]       ) == no_batch_dims:
																												UpperCamelCase    :	Optional[int]   	=		t.expand(orig_batch_dims + t.shape[no_batch_dims:]       )
																					UpperCamelCase    :	List[str]   	=		t.reshape(-1							,			*t.shape[no_batch_dims:]       )
														else:
																					UpperCamelCase    :	Dict   	=		t.expand(orig_batch_dims + t.shape[no_batch_dims:]       )
														return t

							UpperCamelCase    :	Dict[str, Any]   	=		tensor_tree_map(_prep_inputs							,			SCREAMING_SNAKE_CASE_       )
							UpperCamelCase    :	Union[str, Any]   	=		None
							if _out is not None:
														UpperCamelCase    :	str   	=		tensor_tree_map(lambda SCREAMING_SNAKE_CASE_       : t.view([-1] + list(t.shape[no_batch_dims:]       )       )							,			_out       )

							UpperCamelCase    :	str   	=		1
							for d in orig_batch_dims:
														flat_batch_dim *= d

							UpperCamelCase    :	Optional[Any]   	=		flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

							def _select_chunk(SCREAMING_SNAKE_CASE_      :		torch.Tensor       ) -> torch.Tensor:
														return t[i : i + chunk_size] if t.shape[0] != 1 else t

							UpperCamelCase    :	Optional[int]   	=		0
							UpperCamelCase    :	Tuple   	=		prepped_outputs
							for _ in range(SCREAMING_SNAKE_CASE_       ):
														# Chunk the input
														if not low_mem:
																					UpperCamelCase    :	Optional[Any]   	=		_select_chunk
														else:
																					UpperCamelCase    :	Optional[Any]   	=		partial(
																					    _chunk_slice							,			flat_start=SCREAMING_SNAKE_CASE_							,			flat_end=min(SCREAMING_SNAKE_CASE_							,			i + chunk_size       )							,			no_batch_dims=len(SCREAMING_SNAKE_CASE_       )							,			)

														UpperCamelCase    :	Dict[str, Any]   	=		tensor_tree_map(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       )

														# Run the layer on the chunk
														UpperCamelCase    :	Optional[int]   	=		layer(**SCREAMING_SNAKE_CASE_       )

														# Allocate space for the output
														if out is None:
																					UpperCamelCase    :	Any   	=		tensor_tree_map(lambda SCREAMING_SNAKE_CASE_       : t.new_zeros((flat_batch_dim,) + t.shape[1:]       )							,			SCREAMING_SNAKE_CASE_       )

														# Put the chunk in its pre-allocated space
														if isinstance(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       ):

																					def assign(SCREAMING_SNAKE_CASE_      :		dict							,			SCREAMING_SNAKE_CASE_      :		dict       ) -> None:
																												for k, v in da.items():
																																			if isinstance(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       ):
																																										assign(SCREAMING_SNAKE_CASE_							,			da[k]       )
																																			else:
																																										if _add_into_out:
																																																	v[i : i + chunk_size] += da[k]
																																										else:
																																																	UpperCamelCase    :	Optional[int]   	=		da[k]

																					assign(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       )
														elif isinstance(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       ):
																					for xa, xa in zip(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       ):
																												if _add_into_out:
																																			xa[i : i + chunk_size] += xa
																												else:
																																			UpperCamelCase    :	Union[str, Any]   	=		xa
														elif isinstance(SCREAMING_SNAKE_CASE_							,			torch.Tensor       ):
																					if _add_into_out:
																												out[i : i + chunk_size] += output_chunk
																					else:
																												UpperCamelCase    :	Any   	=		output_chunk
														else:
																					raise ValueError('''Not supported'''       )

														i += chunk_size

							UpperCamelCase    :	Tuple   	=		tensor_tree_map(lambda SCREAMING_SNAKE_CASE_       : t.view(orig_batch_dims + t.shape[1:]       )							,			SCREAMING_SNAKE_CASE_       )

							return out



class    UpperCAmelCase_	:
							'''simple docstring'''






							def __init__(     self ,					__SCREAMING_SNAKE_CASE = 512 ,					):




														"""simple docstring"""




														UpperCamelCase    :	Dict   	=		max_chunk_size
														UpperCamelCase    :	Optional[int]   	=		None
														UpperCamelCase    :	Optional[tuple]   	=		None






							def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):




														"""simple docstring"""




														logging.info('''Tuning chunk size...'''						)

														if min_chunk_size >= self.max_chunk_size:
																					return min_chunk_size

														UpperCamelCase    :	List[int]   	=		[2**l for l in range(int(math.log(self.max_chunk_size ,					2						)						) + 1						)]
														UpperCamelCase    :	List[Any]   	=		[c for c in candidates if c > min_chunk_size]
														UpperCamelCase    :	str   	=		[min_chunk_size] + candidates
														candidates[-1] += 4

														def test_chunk_size(__SCREAMING_SNAKE_CASE						) -> bool:
																					try:
																												with torch.no_grad():
																																			fn(*__SCREAMING_SNAKE_CASE ,					chunk_size=__SCREAMING_SNAKE_CASE						)
																												return True
																					except RuntimeError:
																												return False

														UpperCamelCase    :	Optional[int]   	=		0
														UpperCamelCase    :	List[str]   	=		len(__SCREAMING_SNAKE_CASE						) - 1
														while i > min_viable_chunk_size_index:
																					UpperCamelCase    :	Optional[int]   	=		test_chunk_size(candidates[i]						)
																					if not viable:
																												UpperCamelCase    :	Any   	=		(min_viable_chunk_size_index + i) // 2
																					else:
																												UpperCamelCase    :	Tuple   	=		i
																												UpperCamelCase    :	Optional[int]   	=		(i + len(__SCREAMING_SNAKE_CASE						) - 1) // 2

														return candidates[min_viable_chunk_size_index]






							def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):




														"""simple docstring"""




														UpperCamelCase    :	Optional[Any]   	=		True
														for aa, aa in zip(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):
																					assert type(__SCREAMING_SNAKE_CASE						) == type(__SCREAMING_SNAKE_CASE						)
																					if isinstance(__SCREAMING_SNAKE_CASE ,					(list, tuple)						):
																												consistent &= self._compare_arg_caches(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)
																					elif isinstance(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):
																												UpperCamelCase    :	Union[str, Any]   	=		[v for _, v in sorted(aa.items() ,					key=lambda __SCREAMING_SNAKE_CASE						: x[0]						)]
																												UpperCamelCase    :	Optional[int]   	=		[v for _, v in sorted(aa.items() ,					key=lambda __SCREAMING_SNAKE_CASE						: x[0]						)]
																												consistent &= self._compare_arg_caches(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)
																					else:
																												consistent &= aa == aa

														return consistent






							def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					):




														"""simple docstring"""




														UpperCamelCase    :	List[Any]   	=		True
														UpperCamelCase    :	tuple   	=		tree_map(lambda __SCREAMING_SNAKE_CASE						: a.shape if isinstance(__SCREAMING_SNAKE_CASE ,					torch.Tensor						) else a ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)
														if self.cached_arg_data is not None:
																					# If args have changed shape/value, we need to re-tune
																					assert len(self.cached_arg_data						) == len(__SCREAMING_SNAKE_CASE						)
																					UpperCamelCase    :	Optional[Any]   	=		self._compare_arg_caches(self.cached_arg_data ,					__SCREAMING_SNAKE_CASE						)
														else:
																					# Otherwise, we can reuse the precomputed value
																					UpperCamelCase    :	Optional[Any]   	=		False

														if not consistent:
																					UpperCamelCase    :	Optional[int]   	=		self._determine_favorable_chunk_size(
																					    __SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					)
																					UpperCamelCase    :	Any   	=		arg_data

														assert self.cached_chunk_size is not None

														return self.cached_chunk_size





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase : Any 			=				logging.get_logger(__name__)

__UpperCAmelCase : int 			=				"▁"

__UpperCAmelCase : Tuple 			=				{"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

__UpperCAmelCase : Dict 			=				{
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

__UpperCAmelCase : Dict 			=				{
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

__UpperCAmelCase : str 			=				{
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

__UpperCAmelCase : Optional[int] 			=				{
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}



class    UpperCAmelCase_	(							_a):
       '''simple docstring'''
       __UpperCamelCase      :     List[str]							=					["input_ids"]

       __UpperCamelCase      :     List[str]							=					VOCAB_FILES_NAMES
       __UpperCamelCase      :     List[Any]							=					PRETRAINED_INIT_CONFIGURATION
       __UpperCamelCase      :     List[Any]							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __UpperCamelCase      :     int							=					PRETRAINED_VOCAB_FILES_MAP
       __UpperCamelCase      :     List[str]							=					RESOURCE_FILES_NAMES






       def __init__(     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None ,					__SCREAMING_SNAKE_CASE=False ,					__SCREAMING_SNAKE_CASE="utf8" ,					__SCREAMING_SNAKE_CASE="[UNK]" ,					__SCREAMING_SNAKE_CASE="[SEP]" ,					__SCREAMING_SNAKE_CASE="[PAD]" ,					__SCREAMING_SNAKE_CASE="[CLS]" ,					__SCREAMING_SNAKE_CASE="[MASK]" ,					__SCREAMING_SNAKE_CASE = None ,					**__SCREAMING_SNAKE_CASE ,					):




              """simple docstring"""




              UpperCamelCase    :	int   	=		{} if sp_model_kwargs is None else sp_model_kwargs
              super().__init__(
                  do_lower_case=__SCREAMING_SNAKE_CASE ,					unk_token=__SCREAMING_SNAKE_CASE ,					sep_token=__SCREAMING_SNAKE_CASE ,					pad_token=__SCREAMING_SNAKE_CASE ,					cls_token=__SCREAMING_SNAKE_CASE ,					mask_token=__SCREAMING_SNAKE_CASE ,					vocab_file=__SCREAMING_SNAKE_CASE ,					encoding=__SCREAMING_SNAKE_CASE ,					sp_model_kwargs=self.sp_model_kwargs ,					**__SCREAMING_SNAKE_CASE ,					)
              UpperCamelCase    :	List[str]   	=		do_lower_case
              UpperCamelCase    :	Dict   	=		sentencepiece_model_ckpt
              UpperCamelCase    :	str   	=		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
              self.sp_model.Load(__SCREAMING_SNAKE_CASE						)

              # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
              if vocab_file is not None:
                     UpperCamelCase    :	Optional[Any]   	=		self.load_vocab(filepath=__SCREAMING_SNAKE_CASE						)
              else:
                     UpperCamelCase    :	int   	=		{self.sp_model.id_to_piece(__SCREAMING_SNAKE_CASE						): id for id in range(self.sp_model.get_piece_size()						)}
              UpperCamelCase    :	str   	=		{v: k for k, v in self.vocab.items()}






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if text is None:
                     return None

              UpperCamelCase    :	str   	=		self.tokenize(__SCREAMING_SNAKE_CASE						)
              UpperCamelCase	, UpperCamelCase    :	str   	=		'''''', []

              for i, ch in enumerate(__SCREAMING_SNAKE_CASE						):
                     if ch in self.SP_CHAR_MAPPING:
                            UpperCamelCase    :	Optional[int]   	=		self.SP_CHAR_MAPPING.get(__SCREAMING_SNAKE_CASE						)
                     else:
                            UpperCamelCase    :	Optional[Any]   	=		unicodedata.normalize('''NFKC''' ,					__SCREAMING_SNAKE_CASE						)
                     if self.is_whitespace(__SCREAMING_SNAKE_CASE						):
                            continue
                     normalized_text += ch
                     char_mapping.extend([i] * len(__SCREAMING_SNAKE_CASE						)						)

              UpperCamelCase	, UpperCamelCase	, UpperCamelCase    :	Tuple   	=		normalized_text, [], 0

              if self.do_lower_case:
                     UpperCamelCase    :	Tuple   	=		text.lower()

              for token in split_tokens:
                     if token[:1] == "▁":
                            UpperCamelCase    :	Any   	=		token[1:]
                     UpperCamelCase    :	Optional[int]   	=		text[offset:].index(__SCREAMING_SNAKE_CASE						) + offset
                     UpperCamelCase    :	List[Any]   	=		start + len(__SCREAMING_SNAKE_CASE						)

                     token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)						)
                     UpperCamelCase    :	str   	=		end
              return token_mapping






       @property
       def  _lowercase   (     self						):




              """simple docstring"""




              return len(self.vocab						)






       def  _lowercase   (     self						):




              """simple docstring"""




              return dict(self.vocab ,					**self.added_tokens_encoder						)






       def __getstate__(     self						):




              """simple docstring"""




              UpperCamelCase    :	Optional[Any]   	=		self.__dict__.copy()
              UpperCamelCase    :	str   	=		None
              return state






       def __setstate__(     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	Tuple   	=		d

              # for backward compatibility
              if not hasattr(self ,					'''sp_model_kwargs'''						):
                     UpperCamelCase    :	Optional[int]   	=		{}

              UpperCamelCase    :	Optional[Any]   	=		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
              self.sp_model.Load(self.sentencepiece_model_ckpt						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              return "".join((self.SP_CHAR_MAPPING.get(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						) for c in text)						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=False ,					__SCREAMING_SNAKE_CASE=64 ,					__SCREAMING_SNAKE_CASE=0.1						):




              """simple docstring"""




              if self.sp_model_kwargs.get('''enable_sampling'''						) is True:
                     UpperCamelCase    :	List[str]   	=		True
              if self.sp_model_kwargs.get('''alpha'''						) is not None:
                     UpperCamelCase    :	Any   	=		self.sp_model_kwargs.get('''alpha'''						)
              if self.sp_model_kwargs.get('''nbest_size'''						) is not None:
                     UpperCamelCase    :	Tuple   	=		self.sp_model_kwargs.get('''nbest_size'''						)

              if not enable_sampling:
                     UpperCamelCase    :	int   	=		self.sp_model.EncodeAsPieces(__SCREAMING_SNAKE_CASE						)
              else:
                     UpperCamelCase    :	Optional[Any]   	=		self.sp_model.SampleEncodeAsPieces(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)
              UpperCamelCase    :	List[str]   	=		[]
              for pi, piece in enumerate(__SCREAMING_SNAKE_CASE						):
                     if piece == SPIECE_UNDERLINE:
                            if not pieces[pi + 1].startswith(__SCREAMING_SNAKE_CASE						) and pi != 0:
                                   new_pieces.append(__SCREAMING_SNAKE_CASE						)
                                   continue
                            else:
                                   continue
                     UpperCamelCase    :	Any   	=		0
                     for i, chunk in enumerate(__SCREAMING_SNAKE_CASE						):
                            if chunk == SPIECE_UNDERLINE:
                                   continue
                            if self.is_ch_char(__SCREAMING_SNAKE_CASE						) or self.is_punct(__SCREAMING_SNAKE_CASE						):
                                   if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                          new_pieces.append(piece[lst_i:i]						)
                                   new_pieces.append(__SCREAMING_SNAKE_CASE						)
                                   UpperCamelCase    :	Dict   	=		i + 1
                            elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                                   if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                          new_pieces.append(piece[lst_i:i]						)
                                   UpperCamelCase    :	Union[str, Any]   	=		i
                            elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                                   if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                          new_pieces.append(piece[lst_i:i]						)
                                   UpperCamelCase    :	Any   	=		i
                     if len(__SCREAMING_SNAKE_CASE						) > lst_i:
                            new_pieces.append(piece[lst_i:]						)
              return new_pieces






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	Optional[int]   	=		''''''.join(__SCREAMING_SNAKE_CASE						).replace(__SCREAMING_SNAKE_CASE ,					''' '''						).strip()
              return out_string






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	int   	=		self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE						)
              UpperCamelCase    :	int   	=		''''''.join(__SCREAMING_SNAKE_CASE						).replace(__SCREAMING_SNAKE_CASE ,					''' '''						).strip()
              return out_string






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              return self.vocab.get(__SCREAMING_SNAKE_CASE ,					self.vocab.get(self.unk_token						)						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              return self.reverse_vocab.get(__SCREAMING_SNAKE_CASE ,					self.unk_token						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None						):




              """simple docstring"""




              if token_ids_a is None:
                     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
              UpperCamelCase    :	Any   	=		[self.cls_token_id]
              UpperCamelCase    :	str   	=		[self.sep_token_id]
              return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None						):




              """simple docstring"""




              if offset_mapping_a is None:
                     return [(0, 0)] + offset_mapping_a + [(0, 0)]

              return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None ,					__SCREAMING_SNAKE_CASE=False						):




              """simple docstring"""




              if already_has_special_tokens:
                     if token_ids_a is not None:
                            raise ValueError(
                                '''You should not supply a second sequence if the provided sequence of '''
                                '''ids is already formatted with special tokens for the model.'''						)
                     return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

              if token_ids_a is not None:
                     return [1] + ([0] * len(__SCREAMING_SNAKE_CASE						)) + [1, 1] + ([0] * len(__SCREAMING_SNAKE_CASE						)) + [1]
              return [1] + ([0] * len(__SCREAMING_SNAKE_CASE						)) + [1]






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE = None						):




              """simple docstring"""




              if token_ids_a is None:
                     # [CLS] X [SEP]
                     return (len(__SCREAMING_SNAKE_CASE						) + 2) * [0]

              # [CLS] A [SEP] [SEP] B [SEP]
              return [0] * (len(__SCREAMING_SNAKE_CASE						) + 1) + [1] * (len(__SCREAMING_SNAKE_CASE						) + 3)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if "\u4e00" <= char <= "\u9fff":
                     return True
              return False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if ("a" <= char <= "z") or ("A" <= char <= "Z"):
                     return True
              return False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if char in ",;:.?!~，；：。？！《》【】":
                     return True
              return False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if char == " " or char == "\t" or char == "\n" or char == "\r":
                     return True
              if len(__SCREAMING_SNAKE_CASE						) == 1:
                     UpperCamelCase    :	Optional[int]   	=		unicodedata.category(__SCREAMING_SNAKE_CASE						)
                     if cat == "Zs":
                            return True
              return False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	int   	=		{}
              with io.open(__SCREAMING_SNAKE_CASE ,					'''r''' ,					encoding='''utf-8'''						) as f:
                     for index, line in enumerate(__SCREAMING_SNAKE_CASE						):
                            UpperCamelCase    :	Tuple   	=		line.rstrip('''\n'''						)
                            UpperCamelCase    :	List[Any]   	=		int(__SCREAMING_SNAKE_CASE						)

              return token_to_idx






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE = None						):




              """simple docstring"""




              UpperCamelCase    :	Union[str, Any]   	=		0
              if os.path.isdir(__SCREAMING_SNAKE_CASE						):
                     UpperCamelCase    :	Dict   	=		os.path.join(
                         __SCREAMING_SNAKE_CASE ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
              else:
                     UpperCamelCase    :	Union[str, Any]   	=		(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
              with open(__SCREAMING_SNAKE_CASE ,					'''w''' ,					encoding='''utf-8'''						) as writer:
                     for token, token_index in sorted(self.vocab.items() ,					key=lambda __SCREAMING_SNAKE_CASE						: kv[1]						):
                            if index != token_index:
                                   logger.warning(
                                       f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                                       ''' Please check that the vocabulary is not corrupted!'''						)
                                   UpperCamelCase    :	List[Any]   	=		token_index
                            writer.write(token + '''\n'''						)
                            index += 1

              UpperCamelCase    :	Tuple   	=		os.path.join(__SCREAMING_SNAKE_CASE ,					'''sentencepiece.bpe.model'''						)
              with open(__SCREAMING_SNAKE_CASE ,					'''wb'''						) as fi:
                     UpperCamelCase    :	List[Any]   	=		self.sp_model.serialized_model_proto()
                     fi.write(__SCREAMING_SNAKE_CASE						)

              return (vocab_file,)

