


from __future__ import annotations

import unittest

from transformers import MobileBertConfig, is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TF_MODEL_FOR_PRETRAINING_MAPPING,
      TFMobileBertForMaskedLM,
      TFMobileBertForMultipleChoice,
      TFMobileBertForNextSentencePrediction,
      TFMobileBertForPreTraining,
      TFMobileBertForQuestionAnswering,
      TFMobileBertForSequenceClassification,
      TFMobileBertForTokenClassification,
      TFMobileBertModel,
  )



@require_tf
class    UpperCAmelCase_	(							_a,     _a,     unittest.TestCase):
       '''simple docstring'''
       __UpperCamelCase      :     Dict							=					(
           (
               TFMobileBertModel,
               TFMobileBertForMaskedLM,
               TFMobileBertForNextSentencePrediction,
               TFMobileBertForPreTraining,
               TFMobileBertForQuestionAnswering,
               TFMobileBertForSequenceClassification,
               TFMobileBertForTokenClassification,
               TFMobileBertForMultipleChoice,
           )
           if is_tf_available()
           else ()
       )
       __UpperCamelCase      :     str							=					(
           {
               "feature-extraction": TFMobileBertModel,
               "fill-mask": TFMobileBertForMaskedLM,
               "question-answering": TFMobileBertForQuestionAnswering,
               "text-classification": TFMobileBertForSequenceClassification,
               "token-classification": TFMobileBertForTokenClassification,
               "zero-shot": TFMobileBertForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )
       __UpperCamelCase      :     Union[str, Any]							=					False
       __UpperCamelCase      :     str							=					False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=False						):




              """simple docstring"""




              UpperCamelCase    :	Union[str, Any]   	=		super()._prepare_for_class(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					return_labels=__SCREAMING_SNAKE_CASE						)

              if return_labels:
                     if model_class in get_values(__SCREAMING_SNAKE_CASE						):
                            UpperCamelCase    :	Dict   	=		tf.zeros(self.model_tester.batch_size ,					dtype=tf.intaa						)

              return inputs_dict



       class    UpperCAmelCase_	(							_a):
              '''simple docstring'''






              def __init__(     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=13 ,					__SCREAMING_SNAKE_CASE=7 ,					__SCREAMING_SNAKE_CASE=True ,					__SCREAMING_SNAKE_CASE=True ,					__SCREAMING_SNAKE_CASE=True ,					__SCREAMING_SNAKE_CASE=True ,					__SCREAMING_SNAKE_CASE=99 ,					__SCREAMING_SNAKE_CASE=32 ,					__SCREAMING_SNAKE_CASE=32 ,					__SCREAMING_SNAKE_CASE=2 ,					__SCREAMING_SNAKE_CASE=4 ,					__SCREAMING_SNAKE_CASE=37 ,					__SCREAMING_SNAKE_CASE="gelu" ,					__SCREAMING_SNAKE_CASE=0.1 ,					__SCREAMING_SNAKE_CASE=0.1 ,					__SCREAMING_SNAKE_CASE=512 ,					__SCREAMING_SNAKE_CASE=16 ,					__SCREAMING_SNAKE_CASE=2 ,					__SCREAMING_SNAKE_CASE=0.02 ,					__SCREAMING_SNAKE_CASE=3 ,					__SCREAMING_SNAKE_CASE=4 ,					__SCREAMING_SNAKE_CASE=None ,					):




                     """simple docstring"""




                     UpperCamelCase    :	Tuple   	=		parent
                     UpperCamelCase    :	Any   	=		batch_size
                     UpperCamelCase    :	Optional[Any]   	=		seq_length
                     UpperCamelCase    :	Tuple   	=		is_training
                     UpperCamelCase    :	Dict   	=		use_input_mask
                     UpperCamelCase    :	int   	=		use_token_type_ids
                     UpperCamelCase    :	List[Any]   	=		use_labels
                     UpperCamelCase    :	str   	=		vocab_size
                     UpperCamelCase    :	List[str]   	=		hidden_size
                     UpperCamelCase    :	Dict   	=		num_hidden_layers
                     UpperCamelCase    :	Tuple   	=		num_attention_heads
                     UpperCamelCase    :	Optional[Any]   	=		intermediate_size
                     UpperCamelCase    :	List[Any]   	=		hidden_act
                     UpperCamelCase    :	int   	=		hidden_dropout_prob
                     UpperCamelCase    :	Tuple   	=		attention_probs_dropout_prob
                     UpperCamelCase    :	Union[str, Any]   	=		max_position_embeddings
                     UpperCamelCase    :	Dict   	=		type_vocab_size
                     UpperCamelCase    :	int   	=		type_sequence_label_size
                     UpperCamelCase    :	List[str]   	=		initializer_range
                     UpperCamelCase    :	Tuple   	=		num_labels
                     UpperCamelCase    :	Dict   	=		num_choices
                     UpperCamelCase    :	List[Any]   	=		scope
                     UpperCamelCase    :	Any   	=		embedding_size






              def  _lowercase   (     self						):




                     """simple docstring"""




                     UpperCamelCase    :	int   	=		ids_tensor([self.batch_size, self.seq_length] ,					self.vocab_size						)

                     UpperCamelCase    :	List[str]   	=		None
                     if self.use_input_mask:
                            UpperCamelCase    :	Dict   	=		random_attention_mask([self.batch_size, self.seq_length]						)

                     UpperCamelCase    :	Tuple   	=		None
                     if self.use_token_type_ids:
                            UpperCamelCase    :	Any   	=		ids_tensor([self.batch_size, self.seq_length] ,					self.type_vocab_size						)

                     UpperCamelCase    :	Union[str, Any]   	=		None
                     UpperCamelCase    :	Tuple   	=		None
                     UpperCamelCase    :	Optional[Any]   	=		None
                     if self.use_labels:
                            UpperCamelCase    :	str   	=		ids_tensor([self.batch_size] ,					self.type_sequence_label_size						)
                            UpperCamelCase    :	int   	=		ids_tensor([self.batch_size, self.seq_length] ,					self.num_labels						)
                            UpperCamelCase    :	List[str]   	=		ids_tensor([self.batch_size] ,					self.num_choices						)

                     UpperCamelCase    :	str   	=		MobileBertConfig(
                         vocab_size=self.vocab_size ,					hidden_size=self.hidden_size ,					num_hidden_layers=self.num_hidden_layers ,					num_attention_heads=self.num_attention_heads ,					intermediate_size=self.intermediate_size ,					hidden_act=self.hidden_act ,					hidden_dropout_prob=self.hidden_dropout_prob ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob ,					max_position_embeddings=self.max_position_embeddings ,					type_vocab_size=self.type_vocab_size ,					initializer_range=self.initializer_range ,					embedding_size=self.embedding_size ,					)

                     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






              def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):




                     """simple docstring"""




                     UpperCamelCase    :	Tuple   	=		TFMobileBertModel(config=__SCREAMING_SNAKE_CASE						)
                     UpperCamelCase    :	Optional[Any]   	=		{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
                     UpperCamelCase    :	Union[str, Any]   	=		model(__SCREAMING_SNAKE_CASE						)

                     UpperCamelCase    :	Optional[int]   	=		[input_ids, input_mask]
                     UpperCamelCase    :	Optional[int]   	=		model(__SCREAMING_SNAKE_CASE						)

                     UpperCamelCase    :	Dict   	=		model(__SCREAMING_SNAKE_CASE						)

                     self.parent.assertEqual(
                         result.last_hidden_state.shape ,					(self.batch_size, self.seq_length, self.hidden_size)						)
                     self.parent.assertEqual(result.pooler_output.shape ,					(self.batch_size, self.hidden_size)						)






              def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):




                     """simple docstring"""




                     UpperCamelCase    :	Tuple   	=		TFMobileBertForMaskedLM(config=__SCREAMING_SNAKE_CASE						)
                     UpperCamelCase    :	List[Any]   	=		{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
                     UpperCamelCase    :	int   	=		model(__SCREAMING_SNAKE_CASE						)
                     self.parent.assertEqual(result.logits.shape ,					(self.batch_size, self.seq_length, self.vocab_size)						)






              def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):




                     """simple docstring"""




                     UpperCamelCase    :	Tuple   	=		TFMobileBertForNextSentencePrediction(config=__SCREAMING_SNAKE_CASE						)
                     UpperCamelCase    :	int   	=		{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
                     UpperCamelCase    :	List[str]   	=		model(__SCREAMING_SNAKE_CASE						)
                     self.parent.assertEqual(result.logits.shape ,					(self.batch_size, 2)						)






              def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):




                     """simple docstring"""




                     UpperCamelCase    :	Dict   	=		TFMobileBertForPreTraining(config=__SCREAMING_SNAKE_CASE						)
                     UpperCamelCase    :	Optional[Any]   	=		{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
                     UpperCamelCase    :	Tuple   	=		model(__SCREAMING_SNAKE_CASE						)
                     self.parent.assertEqual(
                         result.prediction_logits.shape ,					(self.batch_size, self.seq_length, self.vocab_size)						)
                     self.parent.assertEqual(result.seq_relationship_logits.shape ,					(self.batch_size, 2)						)






              def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):




                     """simple docstring"""




                     UpperCamelCase    :	Optional[Any]   	=		self.num_labels
                     UpperCamelCase    :	str   	=		TFMobileBertForSequenceClassification(config=__SCREAMING_SNAKE_CASE						)
                     UpperCamelCase    :	Optional[int]   	=		{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
                     UpperCamelCase    :	int   	=		model(__SCREAMING_SNAKE_CASE						)
                     self.parent.assertEqual(result.logits.shape ,					(self.batch_size, self.num_labels)						)






              def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):




                     """simple docstring"""




                     UpperCamelCase    :	Dict   	=		self.num_choices
                     UpperCamelCase    :	Any   	=		TFMobileBertForMultipleChoice(config=__SCREAMING_SNAKE_CASE						)
                     UpperCamelCase    :	Tuple   	=		tf.tile(tf.expand_dims(__SCREAMING_SNAKE_CASE ,					1						) ,					(1, self.num_choices, 1)						)
                     UpperCamelCase    :	Dict   	=		tf.tile(tf.expand_dims(__SCREAMING_SNAKE_CASE ,					1						) ,					(1, self.num_choices, 1)						)
                     UpperCamelCase    :	List[Any]   	=		tf.tile(tf.expand_dims(__SCREAMING_SNAKE_CASE ,					1						) ,					(1, self.num_choices, 1)						)
                     UpperCamelCase    :	int   	=		{
                         '''input_ids''': multiple_choice_inputs_ids,
                         '''attention_mask''': multiple_choice_input_mask,
                         '''token_type_ids''': multiple_choice_token_type_ids,
                     }
                     UpperCamelCase    :	List[Any]   	=		model(__SCREAMING_SNAKE_CASE						)
                     self.parent.assertEqual(result.logits.shape ,					(self.batch_size, self.num_choices)						)






              def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):




                     """simple docstring"""




                     UpperCamelCase    :	Dict   	=		self.num_labels
                     UpperCamelCase    :	str   	=		TFMobileBertForTokenClassification(config=__SCREAMING_SNAKE_CASE						)
                     UpperCamelCase    :	str   	=		{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
                     UpperCamelCase    :	Optional[Any]   	=		model(__SCREAMING_SNAKE_CASE						)
                     self.parent.assertEqual(result.logits.shape ,					(self.batch_size, self.seq_length, self.num_labels)						)






              def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):




                     """simple docstring"""




                     UpperCamelCase    :	Tuple   	=		TFMobileBertForQuestionAnswering(config=__SCREAMING_SNAKE_CASE						)
                     UpperCamelCase    :	Dict   	=		{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
                     UpperCamelCase    :	str   	=		model(__SCREAMING_SNAKE_CASE						)
                     self.parent.assertEqual(result.start_logits.shape ,					(self.batch_size, self.seq_length)						)
                     self.parent.assertEqual(result.end_logits.shape ,					(self.batch_size, self.seq_length)						)






              def  _lowercase   (     self						):




                     """simple docstring"""




                     UpperCamelCase    :	List[Any]   	=		self.prepare_config_and_inputs()
                     (
                         (
                         UpperCamelCase
                     )	, (
                         UpperCamelCase
                     )	, (
                         UpperCamelCase
                     )	, (
                         UpperCamelCase
                     )	, (
                         UpperCamelCase
                     )	, (
                         UpperCamelCase
                     )	, (
                         UpperCamelCase
                     )	, 
                     )    :	Any   	=		config_and_inputs
                     UpperCamelCase    :	Tuple   	=		{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
                     return config, inputs_dict






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	int   	=		TFMobileBertModelTest.TFMobileBertModelTester(self						)
              UpperCamelCase    :	int   	=		ConfigTester(self ,					config_class=__SCREAMING_SNAKE_CASE ,					hidden_size=37						)






       def  _lowercase   (     self						):




              """simple docstring"""




              self.config_tester.run_common_tests()






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	List[Any]   	=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_model(*__SCREAMING_SNAKE_CASE						)






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	Dict   	=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_masked_lm(*__SCREAMING_SNAKE_CASE						)






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	Dict   	=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_multiple_choice(*__SCREAMING_SNAKE_CASE						)






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	List[Any]   	=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*__SCREAMING_SNAKE_CASE						)






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	Optional[int]   	=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_pretraining(*__SCREAMING_SNAKE_CASE						)






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	Optional[int]   	=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_question_answering(*__SCREAMING_SNAKE_CASE						)






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	Optional[Any]   	=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_sequence_classification(*__SCREAMING_SNAKE_CASE						)






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	Tuple   	=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_mobilebert_for_token_classification(*__SCREAMING_SNAKE_CASE						)






       @slow
       def  _lowercase   (     self						):




              """simple docstring"""




              for model_name in ["google/mobilebert-uncased"]:
                     UpperCamelCase    :	Tuple   	=		TFMobileBertModel.from_pretrained(__SCREAMING_SNAKE_CASE						)
                     self.assertIsNotNone(__SCREAMING_SNAKE_CASE						)




@require_tf
class    UpperCAmelCase_	(							unittest.TestCase):
       '''simple docstring'''






       @slow
       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	List[Any]   	=		TFMobileBertForPreTraining.from_pretrained('''google/mobilebert-uncased'''						)
              UpperCamelCase    :	Any   	=		tf.constant([[0, 1, 2, 3, 4, 5]]						)
              UpperCamelCase    :	int   	=		model(__SCREAMING_SNAKE_CASE						)[0]

              UpperCamelCase    :	List[Any]   	=		[1, 6, 30_522]
              self.assertEqual(output.shape ,					__SCREAMING_SNAKE_CASE						)

              UpperCamelCase    :	List[str]   	=		tf.constant(
                  [
                      [
                          [-4.5_919_547, -9.248_295, -9.645_256],
                          [-6.7_306_175, -6.440_284, -6.6_052_837],
                          [-7.2_743_506, -6.7_847_915, -6.024_673],
                      ]
                  ]						)
              tf.debugging.assert_near(output[:, :3, :3] ,					__SCREAMING_SNAKE_CASE ,					atol=1e-4						)





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase : Any 			=				logging.get_logger(__name__)

__UpperCAmelCase : int 			=				"▁"

__UpperCAmelCase : Tuple 			=				{"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

__UpperCAmelCase : Dict 			=				{
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

__UpperCAmelCase : Dict 			=				{
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

__UpperCAmelCase : str 			=				{
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

__UpperCAmelCase : Optional[int] 			=				{
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}



class    UpperCAmelCase_	(							_a):
       '''simple docstring'''
       __UpperCamelCase      :     List[str]							=					["input_ids"]

       __UpperCamelCase      :     List[str]							=					VOCAB_FILES_NAMES
       __UpperCamelCase      :     List[Any]							=					PRETRAINED_INIT_CONFIGURATION
       __UpperCamelCase      :     List[Any]							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __UpperCamelCase      :     int							=					PRETRAINED_VOCAB_FILES_MAP
       __UpperCamelCase      :     List[str]							=					RESOURCE_FILES_NAMES






       def __init__(     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None ,					__SCREAMING_SNAKE_CASE=False ,					__SCREAMING_SNAKE_CASE="utf8" ,					__SCREAMING_SNAKE_CASE="[UNK]" ,					__SCREAMING_SNAKE_CASE="[SEP]" ,					__SCREAMING_SNAKE_CASE="[PAD]" ,					__SCREAMING_SNAKE_CASE="[CLS]" ,					__SCREAMING_SNAKE_CASE="[MASK]" ,					__SCREAMING_SNAKE_CASE = None ,					**__SCREAMING_SNAKE_CASE ,					):




              """simple docstring"""




              UpperCamelCase    :	int   	=		{} if sp_model_kwargs is None else sp_model_kwargs
              super().__init__(
                  do_lower_case=__SCREAMING_SNAKE_CASE ,					unk_token=__SCREAMING_SNAKE_CASE ,					sep_token=__SCREAMING_SNAKE_CASE ,					pad_token=__SCREAMING_SNAKE_CASE ,					cls_token=__SCREAMING_SNAKE_CASE ,					mask_token=__SCREAMING_SNAKE_CASE ,					vocab_file=__SCREAMING_SNAKE_CASE ,					encoding=__SCREAMING_SNAKE_CASE ,					sp_model_kwargs=self.sp_model_kwargs ,					**__SCREAMING_SNAKE_CASE ,					)
              UpperCamelCase    :	List[str]   	=		do_lower_case
              UpperCamelCase    :	Dict   	=		sentencepiece_model_ckpt
              UpperCamelCase    :	str   	=		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
              self.sp_model.Load(__SCREAMING_SNAKE_CASE						)

              # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
              if vocab_file is not None:
                     UpperCamelCase    :	Optional[Any]   	=		self.load_vocab(filepath=__SCREAMING_SNAKE_CASE						)
              else:
                     UpperCamelCase    :	int   	=		{self.sp_model.id_to_piece(__SCREAMING_SNAKE_CASE						): id for id in range(self.sp_model.get_piece_size()						)}
              UpperCamelCase    :	str   	=		{v: k for k, v in self.vocab.items()}






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if text is None:
                     return None

              UpperCamelCase    :	str   	=		self.tokenize(__SCREAMING_SNAKE_CASE						)
              UpperCamelCase	, UpperCamelCase    :	str   	=		'''''', []

              for i, ch in enumerate(__SCREAMING_SNAKE_CASE						):
                     if ch in self.SP_CHAR_MAPPING:
                            UpperCamelCase    :	Optional[int]   	=		self.SP_CHAR_MAPPING.get(__SCREAMING_SNAKE_CASE						)
                     else:
                            UpperCamelCase    :	Optional[Any]   	=		unicodedata.normalize('''NFKC''' ,					__SCREAMING_SNAKE_CASE						)
                     if self.is_whitespace(__SCREAMING_SNAKE_CASE						):
                            continue
                     normalized_text += ch
                     char_mapping.extend([i] * len(__SCREAMING_SNAKE_CASE						)						)

              UpperCamelCase	, UpperCamelCase	, UpperCamelCase    :	Tuple   	=		normalized_text, [], 0

              if self.do_lower_case:
                     UpperCamelCase    :	Tuple   	=		text.lower()

              for token in split_tokens:
                     if token[:1] == "▁":
                            UpperCamelCase    :	Any   	=		token[1:]
                     UpperCamelCase    :	Optional[int]   	=		text[offset:].index(__SCREAMING_SNAKE_CASE						) + offset
                     UpperCamelCase    :	List[Any]   	=		start + len(__SCREAMING_SNAKE_CASE						)

                     token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)						)
                     UpperCamelCase    :	str   	=		end
              return token_mapping






       @property
       def  _lowercase   (     self						):




              """simple docstring"""




              return len(self.vocab						)






       def  _lowercase   (     self						):




              """simple docstring"""




              return dict(self.vocab ,					**self.added_tokens_encoder						)






       def __getstate__(     self						):




              """simple docstring"""




              UpperCamelCase    :	Optional[Any]   	=		self.__dict__.copy()
              UpperCamelCase    :	str   	=		None
              return state






       def __setstate__(     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	Tuple   	=		d

              # for backward compatibility
              if not hasattr(self ,					'''sp_model_kwargs'''						):
                     UpperCamelCase    :	Optional[int]   	=		{}

              UpperCamelCase    :	Optional[Any]   	=		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
              self.sp_model.Load(self.sentencepiece_model_ckpt						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              return "".join((self.SP_CHAR_MAPPING.get(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						) for c in text)						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=False ,					__SCREAMING_SNAKE_CASE=64 ,					__SCREAMING_SNAKE_CASE=0.1						):




              """simple docstring"""




              if self.sp_model_kwargs.get('''enable_sampling'''						) is True:
                     UpperCamelCase    :	List[str]   	=		True
              if self.sp_model_kwargs.get('''alpha'''						) is not None:
                     UpperCamelCase    :	Any   	=		self.sp_model_kwargs.get('''alpha'''						)
              if self.sp_model_kwargs.get('''nbest_size'''						) is not None:
                     UpperCamelCase    :	Tuple   	=		self.sp_model_kwargs.get('''nbest_size'''						)

              if not enable_sampling:
                     UpperCamelCase    :	int   	=		self.sp_model.EncodeAsPieces(__SCREAMING_SNAKE_CASE						)
              else:
                     UpperCamelCase    :	Optional[Any]   	=		self.sp_model.SampleEncodeAsPieces(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)
              UpperCamelCase    :	List[str]   	=		[]
              for pi, piece in enumerate(__SCREAMING_SNAKE_CASE						):
                     if piece == SPIECE_UNDERLINE:
                            if not pieces[pi + 1].startswith(__SCREAMING_SNAKE_CASE						) and pi != 0:
                                   new_pieces.append(__SCREAMING_SNAKE_CASE						)
                                   continue
                            else:
                                   continue
                     UpperCamelCase    :	Any   	=		0
                     for i, chunk in enumerate(__SCREAMING_SNAKE_CASE						):
                            if chunk == SPIECE_UNDERLINE:
                                   continue
                            if self.is_ch_char(__SCREAMING_SNAKE_CASE						) or self.is_punct(__SCREAMING_SNAKE_CASE						):
                                   if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                          new_pieces.append(piece[lst_i:i]						)
                                   new_pieces.append(__SCREAMING_SNAKE_CASE						)
                                   UpperCamelCase    :	Dict   	=		i + 1
                            elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                                   if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                          new_pieces.append(piece[lst_i:i]						)
                                   UpperCamelCase    :	Union[str, Any]   	=		i
                            elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                                   if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                          new_pieces.append(piece[lst_i:i]						)
                                   UpperCamelCase    :	Any   	=		i
                     if len(__SCREAMING_SNAKE_CASE						) > lst_i:
                            new_pieces.append(piece[lst_i:]						)
              return new_pieces






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	Optional[int]   	=		''''''.join(__SCREAMING_SNAKE_CASE						).replace(__SCREAMING_SNAKE_CASE ,					''' '''						).strip()
              return out_string






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	int   	=		self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE						)
              UpperCamelCase    :	int   	=		''''''.join(__SCREAMING_SNAKE_CASE						).replace(__SCREAMING_SNAKE_CASE ,					''' '''						).strip()
              return out_string






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              return self.vocab.get(__SCREAMING_SNAKE_CASE ,					self.vocab.get(self.unk_token						)						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              return self.reverse_vocab.get(__SCREAMING_SNAKE_CASE ,					self.unk_token						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None						):




              """simple docstring"""




              if token_ids_a is None:
                     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
              UpperCamelCase    :	Any   	=		[self.cls_token_id]
              UpperCamelCase    :	str   	=		[self.sep_token_id]
              return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None						):




              """simple docstring"""




              if offset_mapping_a is None:
                     return [(0, 0)] + offset_mapping_a + [(0, 0)]

              return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None ,					__SCREAMING_SNAKE_CASE=False						):




              """simple docstring"""




              if already_has_special_tokens:
                     if token_ids_a is not None:
                            raise ValueError(
                                '''You should not supply a second sequence if the provided sequence of '''
                                '''ids is already formatted with special tokens for the model.'''						)
                     return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

              if token_ids_a is not None:
                     return [1] + ([0] * len(__SCREAMING_SNAKE_CASE						)) + [1, 1] + ([0] * len(__SCREAMING_SNAKE_CASE						)) + [1]
              return [1] + ([0] * len(__SCREAMING_SNAKE_CASE						)) + [1]






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE = None						):




              """simple docstring"""




              if token_ids_a is None:
                     # [CLS] X [SEP]
                     return (len(__SCREAMING_SNAKE_CASE						) + 2) * [0]

              # [CLS] A [SEP] [SEP] B [SEP]
              return [0] * (len(__SCREAMING_SNAKE_CASE						) + 1) + [1] * (len(__SCREAMING_SNAKE_CASE						) + 3)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if "\u4e00" <= char <= "\u9fff":
                     return True
              return False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if ("a" <= char <= "z") or ("A" <= char <= "Z"):
                     return True
              return False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if char in ",;:.?!~，；：。？！《》【】":
                     return True
              return False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if char == " " or char == "\t" or char == "\n" or char == "\r":
                     return True
              if len(__SCREAMING_SNAKE_CASE						) == 1:
                     UpperCamelCase    :	Optional[int]   	=		unicodedata.category(__SCREAMING_SNAKE_CASE						)
                     if cat == "Zs":
                            return True
              return False






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	int   	=		{}
              with io.open(__SCREAMING_SNAKE_CASE ,					'''r''' ,					encoding='''utf-8'''						) as f:
                     for index, line in enumerate(__SCREAMING_SNAKE_CASE						):
                            UpperCamelCase    :	Tuple   	=		line.rstrip('''\n'''						)
                            UpperCamelCase    :	List[Any]   	=		int(__SCREAMING_SNAKE_CASE						)

              return token_to_idx






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE = None						):




              """simple docstring"""




              UpperCamelCase    :	Union[str, Any]   	=		0
              if os.path.isdir(__SCREAMING_SNAKE_CASE						):
                     UpperCamelCase    :	Dict   	=		os.path.join(
                         __SCREAMING_SNAKE_CASE ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
              else:
                     UpperCamelCase    :	Union[str, Any]   	=		(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
              with open(__SCREAMING_SNAKE_CASE ,					'''w''' ,					encoding='''utf-8'''						) as writer:
                     for token, token_index in sorted(self.vocab.items() ,					key=lambda __SCREAMING_SNAKE_CASE						: kv[1]						):
                            if index != token_index:
                                   logger.warning(
                                       f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                                       ''' Please check that the vocabulary is not corrupted!'''						)
                                   UpperCamelCase    :	List[Any]   	=		token_index
                            writer.write(token + '''\n'''						)
                            index += 1

              UpperCamelCase    :	Tuple   	=		os.path.join(__SCREAMING_SNAKE_CASE ,					'''sentencepiece.bpe.model'''						)
              with open(__SCREAMING_SNAKE_CASE ,					'''wb'''						) as fi:
                     UpperCamelCase    :	List[Any]   	=		self.sp_model.serialized_model_proto()
                     fi.write(__SCREAMING_SNAKE_CASE						)

              return (vocab_file,)

