
"""simple docstring"""




import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A__      :	Tuple          =    logging.get_logger(__name__)

A__      :	Optional[int]          =    """▁"""

A__      :	Any          =    {"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

A__      :	Dict          =    {
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

A__      :	Union[str, Any]          =    {
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

A__      :	int          =    {
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

A__      :	List[Any]          =    {
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}







class   _lowercase						(       lowerCAmelCase_				):


     '''simple docstring'''





     _A								=							["input_ids"]

     _A								=							VOCAB_FILES_NAMES
     _A								=							PRETRAINED_INIT_CONFIGURATION
     _A								=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _A								=							PRETRAINED_VOCAB_FILES_MAP
     _A								=							RESOURCE_FILES_NAMES



     def __init__(      self ,			__UpperCamelCase ,			__UpperCamelCase=None ,			__UpperCamelCase=False ,			__UpperCamelCase="utf8" ,			__UpperCamelCase="[UNK]" ,			__UpperCamelCase="[SEP]" ,			__UpperCamelCase="[PAD]" ,			__UpperCamelCase="[CLS]" ,			__UpperCamelCase="[MASK]" ,			__UpperCamelCase = None ,			**__UpperCamelCase ,			)->					None:
          # Mask token behave like a normal word, i.e. include the space before it and
          # is included in the raw text, there should be a match in a non-normalized sentence.

          UpperCAmelCase__							: int 							=						{} if sp_model_kwargs is None else sp_model_kwargs
          super().__init__(
              do_lower_case=__UpperCamelCase ,			unk_token=__UpperCamelCase ,			sep_token=__UpperCamelCase ,			pad_token=__UpperCamelCase ,			cls_token=__UpperCamelCase ,			mask_token=__UpperCamelCase ,			vocab_file=__UpperCamelCase ,			encoding=__UpperCamelCase ,			sp_model_kwargs=self.sp_model_kwargs ,			**__UpperCamelCase ,			)
          UpperCAmelCase__							: Dict 							=						do_lower_case
          UpperCAmelCase__							: Dict 							=						sentencepiece_model_ckpt
          UpperCAmelCase__							: Any 							=						spm.SentencePieceProcessor(**self.sp_model_kwargs  )
          self.sp_model.Load(__UpperCamelCase  )

          # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
          if vocab_file is not None:
               UpperCAmelCase__							: List[str] 							=						self.load_vocab(filepath=__UpperCamelCase  )
          else:
               UpperCAmelCase__							: str 							=						{self.sp_model.id_to_piece(__UpperCamelCase  ): id for id in range(self.sp_model.get_piece_size()  )}
          UpperCAmelCase__							: Tuple 							=						{v: k for k, v in self.vocab.items()}



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					int:
          if text is None:
               return None

          UpperCAmelCase__							: Dict 							=						self.tokenize(__UpperCamelCase  )
          UpperCAmelCase__							,	UpperCAmelCase__							: Optional[int] 							=						"", []

          for i, ch in enumerate(__UpperCamelCase  ):
               if ch in self.SP_CHAR_MAPPING:
                    UpperCAmelCase__							: Any 							=						self.SP_CHAR_MAPPING.get(__UpperCamelCase  )
               else:
                    UpperCAmelCase__							: Union[str, Any] 							=						unicodedata.normalize("NFKC" ,			__UpperCamelCase  )
               if self.is_whitespace(__UpperCamelCase  ):
                    continue
               normalized_text += ch
               char_mapping.extend([i] * len(__UpperCamelCase  )  )

          UpperCAmelCase__							,	UpperCAmelCase__							,	UpperCAmelCase__							: Tuple 							=						normalized_text, [], 0

          if self.do_lower_case:
               UpperCAmelCase__							: Optional[Any] 							=						text.lower()

          for token in split_tokens:
               if token[:1] == "▁":
                    UpperCAmelCase__							: Union[str, Any] 							=						token[1:]
               UpperCAmelCase__							: Optional[int] 							=						text[offset:].index(__UpperCamelCase  ) + offset
               UpperCAmelCase__							: Optional[Any] 							=						start + len(__UpperCamelCase  )

               token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)  )
               UpperCAmelCase__							: List[str] 							=						end
          return token_mapping



     @property
     def  lowerCAmelCase__       (      self  )->					int:
          return len(self.vocab  )



     def  lowerCAmelCase__       (      self  )->					Optional[int]:
          return dict(self.vocab ,			**self.added_tokens_encoder  )



     def __getstate__(      self  )->					Dict:
          UpperCAmelCase__							: Any 							=						self.__dict__.copy()
          UpperCAmelCase__							: List[Any] 							=						None
          return state



     def __setstate__(      self ,			__UpperCamelCase  )->					Optional[Any]:
          UpperCAmelCase__							: List[str] 							=						d

          # for backward compatibility
          if not hasattr(self ,			"sp_model_kwargs"  ):
               UpperCAmelCase__							: int 							=						{}

          UpperCAmelCase__							: int 							=						spm.SentencePieceProcessor(**self.sp_model_kwargs  )
          self.sp_model.Load(self.sentencepiece_model_ckpt  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					int:
          return "".join((self.SP_CHAR_MAPPING.get(__UpperCamelCase ,			__UpperCamelCase  ) for c in text)  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase=False ,			__UpperCamelCase=64 ,			__UpperCamelCase=0.1  )->					str:

          if self.sp_model_kwargs.get("enable_sampling"  ) is True:
               UpperCAmelCase__							: Union[str, Any] 							=						True
          if self.sp_model_kwargs.get("alpha"  ) is not None:
               UpperCAmelCase__							: Any 							=						self.sp_model_kwargs.get("alpha"  )
          if self.sp_model_kwargs.get("nbest_size"  ) is not None:
               UpperCAmelCase__							: Tuple 							=						self.sp_model_kwargs.get("nbest_size"  )

          if not enable_sampling:
               UpperCAmelCase__							: List[str] 							=						self.sp_model.EncodeAsPieces(__UpperCamelCase  )
          else:
               UpperCAmelCase__							: Tuple 							=						self.sp_model.SampleEncodeAsPieces(__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )
          UpperCAmelCase__							: List[str] 							=						[]
          for pi, piece in enumerate(__UpperCamelCase  ):
               if piece == SPIECE_UNDERLINE:
                    if not pieces[pi + 1].startswith(__UpperCamelCase  ) and pi != 0:
                         new_pieces.append(__UpperCamelCase  )
                         continue
                    else:
                         continue
               UpperCAmelCase__							: Dict 							=						0
               for i, chunk in enumerate(__UpperCamelCase  ):
                    if chunk == SPIECE_UNDERLINE:
                         continue
                    if self.is_ch_char(__UpperCamelCase  ) or self.is_punct(__UpperCamelCase  ):
                         if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                              new_pieces.append(piece[lst_i:i]  )
                         new_pieces.append(__UpperCamelCase  )
                         UpperCAmelCase__							: Any 							=						i + 1
                    elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                         if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                              new_pieces.append(piece[lst_i:i]  )
                         UpperCAmelCase__							: Union[str, Any] 							=						i
                    elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                         if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                              new_pieces.append(piece[lst_i:i]  )
                         UpperCAmelCase__							: List[Any] 							=						i
               if len(__UpperCamelCase  ) > lst_i:
                    new_pieces.append(piece[lst_i:]  )
          return new_pieces



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Tuple:
          UpperCAmelCase__							: str 							=						"".join(__UpperCamelCase  ).replace(__UpperCamelCase ,			" "  ).strip()
          return out_string



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					int:
          UpperCAmelCase__							: Optional[int] 							=						self.convert_ids_to_tokens(__UpperCamelCase  )
          UpperCAmelCase__							: str 							=						"".join(__UpperCamelCase  ).replace(__UpperCamelCase ,			" "  ).strip()
          return out_string



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Any:
          return self.vocab.get(__UpperCamelCase ,			self.vocab.get(self.unk_token  )  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Union[str, Any]:
          return self.reverse_vocab.get(__UpperCamelCase ,			self.unk_token  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase=None  )->					Tuple:
          if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          UpperCAmelCase__							: Optional[int] 							=						[self.cls_token_id]
          UpperCAmelCase__							: Any 							=						[self.sep_token_id]
          return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase=None  )->					int:
          if offset_mapping_a is None:
               return [(0, 0)] + offset_mapping_a + [(0, 0)]

          return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase=None ,			__UpperCamelCase=False  )->					str:

          if already_has_special_tokens:
               if token_ids_a is not None:
                    raise ValueError(
                        "You should not supply a second sequence if the provided sequence of "
                        "ids is already formatted with special tokens for the model."  )
               return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

          if token_ids_a is not None:
               return [1] + ([0] * len(__UpperCamelCase  )) + [1, 1] + ([0] * len(__UpperCamelCase  )) + [1]
          return [1] + ([0] * len(__UpperCamelCase  )) + [1]



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase = None  )->					List[int]:
          # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
          if token_ids_a is None:
               # [CLS] X [SEP]
               return (len(__UpperCamelCase  ) + 2) * [0]

          # [CLS] A [SEP] [SEP] B [SEP]
          return [0] * (len(__UpperCamelCase  ) + 1) + [1] * (len(__UpperCamelCase  ) + 3)



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Union[str, Any]:
          if "\u4e00" <= char <= "\u9fff":
               return True
          return False



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Union[str, Any]:
          if ("a" <= char <= "z") or ("A" <= char <= "Z"):
               return True
          return False



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Union[str, Any]:
          if char in ",;:.?!~，；：。？！《》【】":
               return True
          return False



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Union[str, Any]:
          if char == " " or char == "\t" or char == "\n" or char == "\r":
               return True
          if len(__UpperCamelCase  ) == 1:
               UpperCAmelCase__							: Tuple 							=						unicodedata.category(__UpperCamelCase  )
               if cat == "Zs":
                    return True
          return False



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Dict:
          UpperCAmelCase__							: int 							=						{}
          with io.open(__UpperCamelCase ,			"r" ,			encoding="utf-8"  ) as f:
               for index, line in enumerate(__UpperCamelCase  ):
                    UpperCAmelCase__							: Any 							=						line.rstrip("\n"  )
                    UpperCAmelCase__							: Union[str, Any] 							=						int(__UpperCamelCase  )

          return token_to_idx



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase = None  )->					Tuple[str]:
          UpperCAmelCase__							: Dict 							=						0
          if os.path.isdir(__UpperCamelCase  ):
               UpperCAmelCase__							: Optional[int] 							=						os.path.join(
                   __UpperCamelCase ,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )
          else:
               UpperCAmelCase__							: Tuple 							=						(filename_prefix + "-" if filename_prefix else "") + save_directory
          with open(__UpperCamelCase ,			"w" ,			encoding="utf-8"  ) as writer:
               for token, token_index in sorted(self.vocab.items() ,			key=lambda __UpperCamelCase  : kv[1]  ):
                    if index != token_index:
                         logger.warning(
                             F"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
                             " Please check that the vocabulary is not corrupted!"  )
                         UpperCAmelCase__							: List[str] 							=						token_index
                    writer.write(token + "\n"  )
                    index += 1

          UpperCAmelCase__							: Any 							=						os.path.join(__UpperCamelCase ,			"sentencepiece.bpe.model"  )
          with open(__UpperCamelCase ,			"wb"  ) as fi:
               UpperCAmelCase__							: Dict 							=						self.sp_model.serialized_model_proto()
               fi.write(__UpperCamelCase  )

          return (vocab_file,)





"""simple docstring"""




import timeit

import numpy as np

import datasets
from datasets.arrow_writer import ArrowWriter
from datasets.features.features import _ArrayXD



def 							a__ (				lowerCAmelCase	:       List[str]		):






     '''simple docstring'''
     def wrapper(*lowerCAmelCase	:       Any       ,    **lowerCAmelCase	:       Tuple		):
          UpperCAmelCase__							: Optional[int] 							=						timeit.default_timer()
          UpperCAmelCase__							: int 							=						func(*lowerCAmelCase       ,    **lowerCAmelCase		)
          UpperCAmelCase__							: List[Any] 							=						timeit.default_timer() - starttime
          return delta

     UpperCAmelCase__							: int 							=						func.__name__

     return wrapper



def 							a__ (				lowerCAmelCase	:       dict       ,    lowerCAmelCase	:       Optional[int]=100       ,    lowerCAmelCase	:       List[str]=None		):






     '''simple docstring'''
     UpperCAmelCase__							: str 							=						[]
     UpperCAmelCase__							: Optional[Any] 							=						seq_shapes or {}
     for i in range(lowerCAmelCase		):
          UpperCAmelCase__							: int 							=						{}
          for col_id, (k, v) in enumerate(features.items()		):
               if isinstance(lowerCAmelCase       ,    _ArrayXD		):
                    UpperCAmelCase__							: List[str] 							=						np.random.rand(*v.shape		).astype(v.dtype		)
               elif isinstance(lowerCAmelCase       ,    datasets.Value		):
                    if v.dtype == "string":
                         UpperCAmelCase__							: Dict 							=						"The small grey turtle was surprisingly fast when challenged."
                    else:
                         UpperCAmelCase__							: str 							=						np.random.randint(10       ,    size=1		).astype(v.dtype		).item()
               elif isinstance(lowerCAmelCase       ,    datasets.Sequence		):
                    while isinstance(lowerCAmelCase       ,    datasets.Sequence		):
                         UpperCAmelCase__							: List[str] 							=						v.feature
                    UpperCAmelCase__							: Optional[int] 							=						seq_shapes[k]
                    UpperCAmelCase__							: Optional[int] 							=						np.random.rand(*lowerCAmelCase		).astype(v.dtype		)
               UpperCAmelCase__							: Union[str, Any] 							=						data

          dummy_data.append((i, example)		)

     return dummy_data



def 							a__ (				lowerCAmelCase	:       List[str]       ,    lowerCAmelCase	:       Tuple       ,    lowerCAmelCase	:       List[str]=100       ,    lowerCAmelCase	:       Optional[int]=None		):






     '''simple docstring'''
     UpperCAmelCase__							: int 							=						generate_examples(lowerCAmelCase       ,    num_examples=lowerCAmelCase       ,    seq_shapes=lowerCAmelCase		)

     with ArrowWriter(features=lowerCAmelCase       ,    path=lowerCAmelCase		) as writer:
          for key, record in dummy_data:
               UpperCAmelCase__							: List[Any] 							=						features.encode_example(lowerCAmelCase		)
               writer.write(lowerCAmelCase		)

          UpperCAmelCase__							,	UpperCAmelCase__							: Optional[Any] 							=						writer.finalize()

     if not num_final_examples == num_examples:
          raise ValueError(
              F"Error writing the dataset, wrote {num_final_examples} examples but should have written {num_examples}."		)

     UpperCAmelCase__							: Optional[int] 							=						datasets.Dataset.from_file(filename=lowerCAmelCase       ,    info=datasets.DatasetInfo(features=lowerCAmelCase		)		)

     return dataset



