


import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase : Optional[Any] 			=				logging.get_logger(__name__)

__UpperCAmelCase : str 			=				{
    "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/config.json",
    "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/config.json",
}



class    UpperCAmelCase_	(							_a):
       '''simple docstring'''
       __UpperCamelCase      :     Any							=					"xlnet"
       __UpperCamelCase      :     Optional[Any]							=					["mems"]
       __UpperCamelCase      :     Dict							=					{
           "n_token": "vocab_size",  # Backward compatibility
           "hidden_size": "d_model",
           "num_attention_heads": "n_head",
           "num_hidden_layers": "n_layer",
       }






       def __init__(     self ,					__SCREAMING_SNAKE_CASE=32_000 ,					__SCREAMING_SNAKE_CASE=1_024 ,					__SCREAMING_SNAKE_CASE=24 ,					__SCREAMING_SNAKE_CASE=16 ,					__SCREAMING_SNAKE_CASE=4_096 ,					__SCREAMING_SNAKE_CASE="gelu" ,					__SCREAMING_SNAKE_CASE=True ,					__SCREAMING_SNAKE_CASE="bi" ,					__SCREAMING_SNAKE_CASE=0.02 ,					__SCREAMING_SNAKE_CASE=1e-12 ,					__SCREAMING_SNAKE_CASE=0.1 ,					__SCREAMING_SNAKE_CASE=512 ,					__SCREAMING_SNAKE_CASE=None ,					__SCREAMING_SNAKE_CASE=True ,					__SCREAMING_SNAKE_CASE=False ,					__SCREAMING_SNAKE_CASE=False ,					__SCREAMING_SNAKE_CASE=-1 ,					__SCREAMING_SNAKE_CASE=False ,					__SCREAMING_SNAKE_CASE="last" ,					__SCREAMING_SNAKE_CASE=True ,					__SCREAMING_SNAKE_CASE="tanh" ,					__SCREAMING_SNAKE_CASE=0.1 ,					__SCREAMING_SNAKE_CASE=5 ,					__SCREAMING_SNAKE_CASE=5 ,					__SCREAMING_SNAKE_CASE=5 ,					__SCREAMING_SNAKE_CASE=1 ,					__SCREAMING_SNAKE_CASE=2 ,					**__SCREAMING_SNAKE_CASE ,					):




              """simple docstring"""




              UpperCamelCase    :	Union[str, Any]   	=		vocab_size
              UpperCamelCase    :	Dict   	=		d_model
              UpperCamelCase    :	str   	=		n_layer
              UpperCamelCase    :	List[Any]   	=		n_head
              if d_model % n_head != 0:
                     raise ValueError(f"""'d_model % n_head' ({d_model % n_head}) should be equal to 0"""						)
              if "d_head" in kwargs:
                     if kwargs["d_head"] != d_model // n_head:
                            raise ValueError(
                                f"""`d_head` ({kwargs["d_head"]}) should be equal to `d_model // n_head` ({d_model // n_head})"""						)
              UpperCamelCase    :	List[str]   	=		d_model // n_head
              UpperCamelCase    :	Optional[Any]   	=		ff_activation
              UpperCamelCase    :	Any   	=		d_inner
              UpperCamelCase    :	Optional[int]   	=		untie_r
              UpperCamelCase    :	str   	=		attn_type

              UpperCamelCase    :	List[str]   	=		initializer_range
              UpperCamelCase    :	int   	=		layer_norm_eps

              UpperCamelCase    :	List[Any]   	=		dropout
              UpperCamelCase    :	Dict   	=		mem_len
              UpperCamelCase    :	Optional[Any]   	=		reuse_len
              UpperCamelCase    :	List[Any]   	=		bi_data
              UpperCamelCase    :	Union[str, Any]   	=		clamp_len
              UpperCamelCase    :	str   	=		same_length

              UpperCamelCase    :	Union[str, Any]   	=		summary_type
              UpperCamelCase    :	List[str]   	=		summary_use_proj
              UpperCamelCase    :	Dict   	=		summary_activation
              UpperCamelCase    :	Union[str, Any]   	=		summary_last_dropout
              UpperCamelCase    :	Tuple   	=		start_n_top
              UpperCamelCase    :	int   	=		end_n_top

              UpperCamelCase    :	Optional[int]   	=		bos_token_id
              UpperCamelCase    :	Union[str, Any]   	=		pad_token_id
              UpperCamelCase    :	str   	=		eos_token_id

              if "use_cache" in kwargs:
                     warnings.warn(
                         '''The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`'''
                         ''' instead.''' ,					__SCREAMING_SNAKE_CASE ,					)
                     UpperCamelCase    :	Any   	=		kwargs['''use_cache''']

              UpperCamelCase    :	Dict   	=		use_mems_eval
              UpperCamelCase    :	Dict   	=		use_mems_train
              super().__init__(pad_token_id=__SCREAMING_SNAKE_CASE ,					bos_token_id=__SCREAMING_SNAKE_CASE ,					eos_token_id=__SCREAMING_SNAKE_CASE ,					**__SCREAMING_SNAKE_CASE						)






       @property
       def  _lowercase   (     self						):




              """simple docstring"""




              logger.info(f"""The model {self.model_type} is one of the few models that has no sequence length limit."""						)
              return -1






       @max_position_embeddings.setter
       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              raise NotImplementedError(
                  f"""The model {self.model_type} is one of the few models that has no sequence length limit."""						)





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase : Any 			=				logging.get_logger(__name__)

__UpperCAmelCase : int 			=				"▁"

__UpperCAmelCase : Tuple 			=				{"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

__UpperCAmelCase : Dict 			=				{
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

__UpperCAmelCase : Dict 			=				{
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

__UpperCAmelCase : str 			=				{
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

__UpperCAmelCase : Optional[int] 			=				{
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}



class    UpperCAmelCase_	(							_a):
       '''simple docstring'''
       __UpperCamelCase      :     List[str]							=					["input_ids"]

       __UpperCamelCase      :     List[str]							=					VOCAB_FILES_NAMES
       __UpperCamelCase      :     List[Any]							=					PRETRAINED_INIT_CONFIGURATION
       __UpperCamelCase      :     List[Any]							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __UpperCamelCase      :     int							=					PRETRAINED_VOCAB_FILES_MAP
       __UpperCamelCase      :     List[str]							=					RESOURCE_FILES_NAMES






       def __init__(     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None ,					__SCREAMING_SNAKE_CASE=False ,					__SCREAMING_SNAKE_CASE="utf8" ,					__SCREAMING_SNAKE_CASE="[UNK]" ,					__SCREAMING_SNAKE_CASE="[SEP]" ,					__SCREAMING_SNAKE_CASE="[PAD]" ,					__SCREAMING_SNAKE_CASE="[CLS]" ,					__SCREAMING_SNAKE_CASE="[MASK]" ,					__SCREAMING_SNAKE_CASE = None ,					**__SCREAMING_SNAKE_CASE ,					):




              """simple docstring"""




              UpperCamelCase    :	int   	=		{} if sp_model_kwargs is None else sp_model_kwargs
              super().__init__(
                  do_lower_case=__SCREAMING_SNAKE_CASE ,					unk_token=__SCREAMING_SNAKE_CASE ,					sep_token=__SCREAMING_SNAKE_CASE ,					pad_token=__SCREAMING_SNAKE_CASE ,					cls_token=__SCREAMING_SNAKE_CASE ,					mask_token=__SCREAMING_SNAKE_CASE ,					vocab_file=__SCREAMING_SNAKE_CASE ,					encoding=__SCREAMING_SNAKE_CASE ,					sp_model_kwargs=self.sp_model_kwargs ,					**__SCREAMING_SNAKE_CASE ,					)
              UpperCamelCase    :	List[str]   	=		do_lower_case
              UpperCamelCase    :	Dict   	=		sentencepiece_model_ckpt
              UpperCamelCase    :	str   	=		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
              self.sp_model.Load(__SCREAMING_SNAKE_CASE						)

              # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
              if vocab_file is not None:
                     UpperCamelCase    :	Optional[Any]   	=		self.load_vocab(filepath=__SCREAMING_SNAKE_CASE						)
              else:
                     UpperCamelCase    :	int   	=		{self.sp_model.id_to_piece(__SCREAMING_SNAKE_CASE						): id for id in range(self.sp_model.get_piece_size()						)}
              UpperCamelCase    :	str   	=		{v: k for k, v in self.vocab.items()}






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if text is None:
                     return None

              UpperCamelCase    :	str   	=		self.tokenize(__SCREAMING_SNAKE_CASE						)
              UpperCamelCase	, UpperCamelCase    :	str   	=		'''''', []

              for i, ch in enumerate(__SCREAMING_SNAKE_CASE						):
                     if ch in self.SP_CHAR_MAPPING:
                            UpperCamelCase    :	Optional[int]   	=		self.SP_CHAR_MAPPING.get(__SCREAMING_SNAKE_CASE						)
                     else:
                            UpperCamelCase    :	Optional[Any]   	=		unicodedata.normalize('''NFKC''' ,					__SCREAMING_SNAKE_CASE						)
                     if self.is_whitespace(__SCREAMING_SNAKE_CASE						):
                            continue
                     normalized_text += ch
                     char_mapping.extend([i] * len(__SCREAMING_SNAKE_CASE						)						)

              UpperCamelCase	, UpperCamelCase	, UpperCamelCase    :	Tuple   	=		normalized_text, [], 0

              if self.do_lower_case:
                     UpperCamelCase    :	Tuple   	=		text.lower()

              for token in split_tokens:
                     if token[:1] == "▁":
                            UpperCamelCase    :	Any   	=		token[1:]
                     UpperCamelCase    :	Optional[int]   	=		text[offset:].index(__SCREAMING_SNAKE_CASE						) + offset
                     UpperCamelCase    :	List[Any]   	=		start + len(__SCREAMING_SNAKE_CASE						)

                     token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)						)
                     UpperCamelCase    :	str   	=		end
              return token_mapping






       @property
       def  _lowercase   (     self						):




              """simple docstring"""




              return len(self.vocab						)






       def  _lowercase   (     self						):




              """simple docstring"""




              return dict(self.vocab ,					**self.added_tokens_encoder						)






       def __getstate__(     self						):




              """simple docstring"""




              UpperCamelCase    :	Optional[Any]   	=		self.__dict__.copy()
              UpperCamelCase    :	str   	=		None
              return state






       def __setstate__(     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	Tuple   	=		d

              # for backward compatibility
              if not hasattr(self ,					'''sp_model_kwargs'''						):
                     UpperCamelCase    :	Optional[int]   	=		{}

              UpperCamelCase    :	Optional[Any]   	=		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
              self.sp_model.Load(self.sentencepiece_model_ckpt						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              return "".join((self.SP_CHAR_MAPPING.get(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						) for c in text)						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=False ,					__SCREAMING_SNAKE_CASE=64 ,					__SCREAMING_SNAKE_CASE=0.1						):




              """simple docstring"""




              if self.sp_model_kwargs.get('''enable_sampling'''						) is True:
                     UpperCamelCase    :	List[str]   	=		True
              if self.sp_model_kwargs.get('''alpha'''						) is not None:
                     UpperCamelCase    :	Any   	=		self.sp_model_kwargs.get('''alpha'''						)
              if self.sp_model_kwargs.get('''nbest_size'''						) is not None:
                     UpperCamelCase    :	Tuple   	=		self.sp_model_kwargs.get('''nbest_size'''						)

              if not enable_sampling:
                     UpperCamelCase    :	int   	=		self.sp_model.EncodeAsPieces(__SCREAMING_SNAKE_CASE						)
              else:
                     UpperCamelCase    :	Optional[Any]   	=		self.sp_model.SampleEncodeAsPieces(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)
              UpperCamelCase    :	List[str]   	=		[]
              for pi, piece in enumerate(__SCREAMING_SNAKE_CASE						):
                     if piece == SPIECE_UNDERLINE:
                            if not pieces[pi + 1].startswith(__SCREAMING_SNAKE_CASE						) and pi != 0:
                                   new_pieces.append(__SCREAMING_SNAKE_CASE						)
                                   continue
                            else:
                                   continue
                     UpperCamelCase    :	Any   	=		0
                     for i, chunk in enumerate(__SCREAMING_SNAKE_CASE						):
                            if chunk == SPIECE_UNDERLINE:
                                   continue
                            if self.is_ch_char(__SCREAMING_SNAKE_CASE						) or self.is_punct(__SCREAMING_SNAKE_CASE						):
                                   if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                          new_pieces.append(piece[lst_i:i]						)
                                   new_pieces.append(__SCREAMING_SNAKE_CASE						)
                                   UpperCamelCase    :	Dict   	=		i + 1
                            elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                                   if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                          new_pieces.append(piece[lst_i:i]						)
                                   UpperCamelCase    :	Union[str, Any]   	=		i
                            elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                                   if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                          new_pieces.append(piece[lst_i:i]						)
                                   UpperCamelCase    :	Any   	=		i
                     if len(__SCREAMING_SNAKE_CASE						) > lst_i:
                            new_pieces.append(piece[lst_i:]						)
              return new_pieces






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	Optional[int]   	=		''''''.join(__SCREAMING_SNAKE_CASE						).replace(__SCREAMING_SNAKE_CASE ,					''' '''						).strip()
              return out_string






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	int   	=		self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE						)
              UpperCamelCase    :	int   	=		''''''.join(__SCREAMING_SNAKE_CASE						).replace(__SCREAMING_SNAKE_CASE ,					''' '''						).strip()
              return out_string






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              return self.vocab.get(__SCREAMING_SNAKE_CASE ,					self.vocab.get(self.unk_token						)						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              return self.reverse_vocab.get(__SCREAMING_SNAKE_CASE ,					self.unk_token						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None						):




              """simple docstring"""




              if token_ids_a is None:
                     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
              UpperCamelCase    :	Any   	=		[self.cls_token_id]
              UpperCamelCase    :	str   	=		[self.sep_token_id]
              return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None						):




              """simple docstring"""




              if offset_mapping_a is None:
                     return [(0, 0)] + offset_mapping_a + [(0, 0)]

              return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None ,					__SCREAMING_SNAKE_CASE=False						):




              """simple docstring"""




              if already_has_special_tokens:
                     if token_ids_a is not None:
                            raise ValueError(
                                '''You should not supply a second sequence if the provided sequence of '''
                                '''ids is already formatted with special tokens for the model.'''						)
                     return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

              if token_ids_a is not None:
                     return [1] + ([0] * len(__SCREAMING_SNAKE_CASE						)) + [1, 1] + ([0] * len(__SCREAMING_SNAKE_CASE						)) + [1]
              return [1] + ([0] * len(__SCREAMING_SNAKE_CASE						)) + [1]






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE = None						):




              """simple docstring"""




              if token_ids_a is None:
                     # [CLS] X [SEP]
                     return (len(__SCREAMING_SNAKE_CASE						) + 2) * [0]

              # [CLS] A [SEP] [SEP] B [SEP]
              return [0] * (len(__SCREAMING_SNAKE_CASE						) + 1) + [1] * (len(__SCREAMING_SNAKE_CASE						) + 3)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if "\u4e00" <= char <= "\u9fff":
                     return True
              return False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if ("a" <= char <= "z") or ("A" <= char <= "Z"):
                     return True
              return False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if char in ",;:.?!~，；：。？！《》【】":
                     return True
              return False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if char == " " or char == "\t" or char == "\n" or char == "\r":
                     return True
              if len(__SCREAMING_SNAKE_CASE						) == 1:
                     UpperCamelCase    :	Optional[int]   	=		unicodedata.category(__SCREAMING_SNAKE_CASE						)
                     if cat == "Zs":
                            return True
              return False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	int   	=		{}
              with io.open(__SCREAMING_SNAKE_CASE ,					'''r''' ,					encoding='''utf-8'''						) as f:
                     for index, line in enumerate(__SCREAMING_SNAKE_CASE						):
                            UpperCamelCase    :	Tuple   	=		line.rstrip('''\n'''						)
                            UpperCamelCase    :	List[Any]   	=		int(__SCREAMING_SNAKE_CASE						)

              return token_to_idx






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE = None						):




              """simple docstring"""




              UpperCamelCase    :	Union[str, Any]   	=		0
              if os.path.isdir(__SCREAMING_SNAKE_CASE						):
                     UpperCamelCase    :	Dict   	=		os.path.join(
                         __SCREAMING_SNAKE_CASE ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
              else:
                     UpperCamelCase    :	Union[str, Any]   	=		(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
              with open(__SCREAMING_SNAKE_CASE ,					'''w''' ,					encoding='''utf-8'''						) as writer:
                     for token, token_index in sorted(self.vocab.items() ,					key=lambda __SCREAMING_SNAKE_CASE						: kv[1]						):
                            if index != token_index:
                                   logger.warning(
                                       f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                                       ''' Please check that the vocabulary is not corrupted!'''						)
                                   UpperCamelCase    :	List[Any]   	=		token_index
                            writer.write(token + '''\n'''						)
                            index += 1

              UpperCamelCase    :	Tuple   	=		os.path.join(__SCREAMING_SNAKE_CASE ,					'''sentencepiece.bpe.model'''						)
              with open(__SCREAMING_SNAKE_CASE ,					'''wb'''						) as fi:
                     UpperCamelCase    :	List[Any]   	=		self.sp_model.serialized_model_proto()
                     fi.write(__SCREAMING_SNAKE_CASE						)

              return (vocab_file,)

