"""simple docstring"""


import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_:					Dict     =					logging.get_logger(__name__)

lowerCAmelCase_:					Dict     =					"▁"

lowerCAmelCase_:					Tuple     =					{"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

lowerCAmelCase_:					Union[str, Any]     =					{
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

lowerCAmelCase_:					Union[str, Any]     =					{
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

lowerCAmelCase_:					str     =					{
    "ernie-m-base": 5_1_4,
    "ernie-m-large": 5_1_4,
}

lowerCAmelCase_:					Optional[int]     =					{
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}






class    a__    (   _a	):
   snake_case_							=      ["input_ids"]

   snake_case_							=      VOCAB_FILES_NAMES
   snake_case_							=      PRETRAINED_INIT_CONFIGURATION
   snake_case_							=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   snake_case_							=      PRETRAINED_VOCAB_FILES_MAP
   snake_case_							=      RESOURCE_FILES_NAMES


   def __init__(     self,				_UpperCAmelCase,				_UpperCAmelCase=None,				_UpperCAmelCase=False,				_UpperCAmelCase="utf8",				_UpperCAmelCase="[UNK]",				_UpperCAmelCase="[SEP]",				_UpperCAmelCase="[PAD]",				_UpperCAmelCase="[CLS]",				_UpperCAmelCase="[MASK]",				_UpperCAmelCase = None,				**_UpperCAmelCase,				):
        '''simple docstring'''



        lowercase__				 =							{} if sp_model_kwargs is None else sp_model_kwargs
        super().__init__(
            do_lower_case=_UpperCAmelCase,				unk_token=_UpperCAmelCase,				sep_token=_UpperCAmelCase,				pad_token=_UpperCAmelCase,				cls_token=_UpperCAmelCase,				mask_token=_UpperCAmelCase,				vocab_file=_UpperCAmelCase,				encoding=_UpperCAmelCase,				sp_model_kwargs=self.sp_model_kwargs,				**_UpperCAmelCase,				)
        lowercase__				 =							do_lower_case
        lowercase__				 =							sentencepiece_model_ckpt
        lowercase__				 =							spm.SentencePieceProcessor(**self.sp_model_kwargs  )
        self.sp_model.Load(_UpperCAmelCase  )

        # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
        if vocab_file is not None:
             lowercase__				 =							self.load_vocab(filepath=_UpperCAmelCase  )
        else:
             lowercase__				 =							{self.sp_model.id_to_piece(_UpperCAmelCase  ): id for id in range(self.sp_model.get_piece_size()  )}
        lowercase__				 =							{v: k for k, v in self.vocab.items()}


   def 	snake_case__	(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        if text is None:
             return None

        lowercase__				 =							self.tokenize(_UpperCAmelCase  )
        lowercase__ ,						lowercase__				 =							"", []

        for i, ch in enumerate(_UpperCAmelCase  ):
             if ch in self.SP_CHAR_MAPPING:
                  lowercase__				 =							self.SP_CHAR_MAPPING.get(_UpperCAmelCase  )
             else:
                  lowercase__				 =							unicodedata.normalize("NFKC",				_UpperCAmelCase  )
             if self.is_whitespace(_UpperCAmelCase  ):
                  continue
             normalized_text += ch
             char_mapping.extend([i] * len(_UpperCAmelCase  )  )

        lowercase__ ,						lowercase__ ,						lowercase__				 =							normalized_text, [], 0

        if self.do_lower_case:
             lowercase__				 =							text.lower()

        for token in split_tokens:
             if token[:1] == "▁":
                  lowercase__				 =							token[1:]
             lowercase__				 =							text[offset:].index(_UpperCAmelCase  ) + offset
             lowercase__				 =							start + len(_UpperCAmelCase  )

             token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)  )
             lowercase__				 =							end
        return token_mapping


   @property
   def 	snake_case__	(     self  ):
        '''simple docstring'''



        return len(self.vocab  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        return dict(self.vocab,				**self.added_tokens_encoder  )


   def __getstate__(     self  ):
        '''simple docstring'''



        lowercase__				 =							self.__dict__.copy()
        lowercase__				 =							None
        return state


   def __setstate__(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        lowercase__				 =							d

        # for backward compatibility
        if not hasattr(self,				"sp_model_kwargs"  ):
             lowercase__				 =							{}

        lowercase__				 =							spm.SentencePieceProcessor(**self.sp_model_kwargs  )
        self.sp_model.Load(self.sentencepiece_model_ckpt  )


   def 	snake_case__	(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        return "".join((self.SP_CHAR_MAPPING.get(_UpperCAmelCase,				_UpperCAmelCase  ) for c in text)  )


   def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase=False,				_UpperCAmelCase=64,				_UpperCAmelCase=0.1  ):
        '''simple docstring'''



        if self.sp_model_kwargs.get("enable_sampling"  ) is True:
             lowercase__				 =							True
        if self.sp_model_kwargs.get("alpha"  ) is not None:
             lowercase__				 =							self.sp_model_kwargs.get("alpha"  )
        if self.sp_model_kwargs.get("nbest_size"  ) is not None:
             lowercase__				 =							self.sp_model_kwargs.get("nbest_size"  )

        if not enable_sampling:
             lowercase__				 =							self.sp_model.EncodeAsPieces(_UpperCAmelCase  )
        else:
             lowercase__				 =							self.sp_model.SampleEncodeAsPieces(_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase  )
        lowercase__				 =							[]
        for pi, piece in enumerate(_UpperCAmelCase  ):
             if piece == SPIECE_UNDERLINE:
                  if not pieces[pi + 1].startswith(_UpperCAmelCase  ) and pi != 0:
                       new_pieces.append(_UpperCAmelCase  )
                       continue
                  else:
                       continue
             lowercase__				 =							0
             for i, chunk in enumerate(_UpperCAmelCase  ):
                  if chunk == SPIECE_UNDERLINE:
                       continue
                  if self.is_ch_char(_UpperCAmelCase  ) or self.is_punct(_UpperCAmelCase  ):
                       if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                            new_pieces.append(piece[lst_i:i]  )
                       new_pieces.append(_UpperCAmelCase  )
                       lowercase__				 =							i + 1
                  elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                       if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                            new_pieces.append(piece[lst_i:i]  )
                       lowercase__				 =							i
                  elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                       if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                            new_pieces.append(piece[lst_i:i]  )
                       lowercase__				 =							i
             if len(_UpperCAmelCase  ) > lst_i:
                  new_pieces.append(piece[lst_i:]  )
        return new_pieces


   def 	snake_case__	(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        lowercase__				 =							"".join(_UpperCAmelCase  ).replace(_UpperCAmelCase,				" "  ).strip()
        return out_string


   def 	snake_case__	(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        lowercase__				 =							self.convert_ids_to_tokens(_UpperCAmelCase  )
        lowercase__				 =							"".join(_UpperCAmelCase  ).replace(_UpperCAmelCase,				" "  ).strip()
        return out_string


   def 	snake_case__	(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        return self.vocab.get(_UpperCAmelCase,				self.vocab.get(self.unk_token  )  )


   def 	snake_case__	(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        return self.reverse_vocab.get(_UpperCAmelCase,				self.unk_token  )


   def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase=None  ):
        '''simple docstring'''



        if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        lowercase__				 =							[self.cls_token_id]
        lowercase__				 =							[self.sep_token_id]
        return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep


   def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase=None  ):
        '''simple docstring'''



        if offset_mapping_a is None:
             return [(0, 0)] + offset_mapping_a + [(0, 0)]

        return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]


   def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase=None,				_UpperCAmelCase=False  ):
        '''simple docstring'''



        if already_has_special_tokens:
             if token_ids_a is not None:
                  raise ValueError(
                      "You should not supply a second sequence if the provided sequence of "
                      "ids is already formatted with special tokens for the model."  )
             return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

        if token_ids_a is not None:
             return [1] + ([0] * len(_UpperCAmelCase  )) + [1, 1] + ([0] * len(_UpperCAmelCase  )) + [1]
        return [1] + ([0] * len(_UpperCAmelCase  )) + [1]


   def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase = None  ):
        '''simple docstring'''



        if token_ids_a is None:
             # [CLS] X [SEP]
             return (len(_UpperCAmelCase  ) + 2) * [0]

        # [CLS] A [SEP] [SEP] B [SEP]
        return [0] * (len(_UpperCAmelCase  ) + 1) + [1] * (len(_UpperCAmelCase  ) + 3)


   def 	snake_case__	(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        if "\u4e00" <= char <= "\u9fff":
             return True
        return False


   def 	snake_case__	(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        if ("a" <= char <= "z") or ("A" <= char <= "Z"):
             return True
        return False


   def 	snake_case__	(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        if char in ",;:.?!~，；：。？！《》【】":
             return True
        return False


   def 	snake_case__	(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        if char == " " or char == "\t" or char == "\n" or char == "\r":
             return True
        if len(_UpperCAmelCase  ) == 1:
             lowercase__				 =							unicodedata.category(_UpperCAmelCase  )
             if cat == "Zs":
                  return True
        return False


   def 	snake_case__	(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        lowercase__				 =							{}
        with io.open(_UpperCAmelCase,				"r",				encoding="utf-8"  ) as f:
             for index, line in enumerate(_UpperCAmelCase  ):
                  lowercase__				 =							line.rstrip("\n"  )
                  lowercase__				 =							int(_UpperCAmelCase  )

        return token_to_idx


   def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase = None  ):
        '''simple docstring'''



        lowercase__				 =							0
        if os.path.isdir(_UpperCAmelCase  ):
             lowercase__				 =							os.path.join(
                 _UpperCAmelCase,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )
        else:
             lowercase__				 =							(filename_prefix + "-" if filename_prefix else "") + save_directory
        with open(_UpperCAmelCase,				"w",				encoding="utf-8"  ) as writer:
             for token, token_index in sorted(self.vocab.items(),				key=lambda _UpperCAmelCase  : kv[1]  ):
                  if index != token_index:
                       logger.warning(
                           F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                           " Please check that the vocabulary is not corrupted!"  )
                       lowercase__				 =							token_index
                  writer.write(token + "\n"  )
                  index += 1

        lowercase__				 =							os.path.join(_UpperCAmelCase,				"sentencepiece.bpe.model"  )
        with open(_UpperCAmelCase,				"wb"  ) as fi:
             lowercase__				 =							self.sp_model.serialized_model_proto()
             fi.write(_UpperCAmelCase  )

        return (vocab_file,)





"""simple docstring"""


import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
 lowerCAmelCase_:					Dict     =					"pt"
elif is_tf_available():
 lowerCAmelCase_:					Dict     =					"tf"
else:
 lowerCAmelCase_:					str     =					"jax"






class    a__    (   _a					,					unittest.TestCase	):
   snake_case_							=      ByTaTokenizer
   snake_case_							=      False


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        super().setUp()
        lowercase__				 =							ByTaTokenizer()
        tokenizer.save_pretrained(self.tmpdirname  )


   @cached_property
   def 	snake_case__	(     self  ):
        '''simple docstring'''



        return ByTaTokenizer.from_pretrained("google/byt5-small"  )


   def 	snake_case__	(     self,				**_UpperCAmelCase  ):
        '''simple docstring'''



        return self.tokenizer_class.from_pretrained(self.tmpdirname,				**_UpperCAmelCase  )


   def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase=False,				_UpperCAmelCase=20,				_UpperCAmelCase=5  ):
        '''simple docstring'''



        lowercase__				 =							[]
        for i in range(len(_UpperCAmelCase  )  ):
             try:
                  lowercase__				 =							tokenizer.decode([i],				clean_up_tokenization_spaces=_UpperCAmelCase  )
             except UnicodeDecodeError:
                  pass
             toks.append((i, tok)  )

        lowercase__				 =							list(filter(lambda _UpperCAmelCase  : re.match(R"^[ a-zA-Z]+$",				t[1]  ),				_UpperCAmelCase  )  )
        lowercase__				 =							list(filter(lambda _UpperCAmelCase  : [t[0]] == tokenizer.encode(t[1],				add_special_tokens=_UpperCAmelCase  ),				_UpperCAmelCase  )  )
        if max_length is not None and len(_UpperCAmelCase  ) > max_length:
             lowercase__				 =							toks[:max_length]
        if min_length is not None and len(_UpperCAmelCase  ) < min_length and len(_UpperCAmelCase  ) > 0:
             while len(_UpperCAmelCase  ) < min_length:
                  lowercase__				 =							toks + toks
        # toks_str = [t[1] for t in toks]
        lowercase__				 =							[t[0] for t in toks]

        # Ensure consistency
        lowercase__				 =							tokenizer.decode(_UpperCAmelCase,				clean_up_tokenization_spaces=_UpperCAmelCase  )
        if " " not in output_txt and len(_UpperCAmelCase  ) > 1:
             lowercase__				 =							(
                 tokenizer.decode([toks_ids[0]],				clean_up_tokenization_spaces=_UpperCAmelCase  )
                 + " "
                 + tokenizer.decode(toks_ids[1:],				clean_up_tokenization_spaces=_UpperCAmelCase  )
             )
        if with_prefix_space:
             lowercase__				 =							" " + output_txt
        lowercase__				 =							tokenizer.encode(_UpperCAmelCase,				add_special_tokens=_UpperCAmelCase  )
        return output_txt, output_ids


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							self.ta_base_tokenizer
        lowercase__				 =							tokenizer(["hi</s>", "I went to the gym</s>", "</s>"]  )
        lowercase__				 =							tokenizer(["hi", "I went to the gym", ""]  )
        self.assertListEqual(batch_with_eos_added["input_ids"],				batch_without_eos_added["input_ids"]  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							self.ta_base_tokenizer
        lowercase__				 =							"Unicode €."
        lowercase__				 =							tokenizer(_UpperCAmelCase  )
        lowercase__				 =							[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
        self.assertEqual(encoded["input_ids"],				_UpperCAmelCase  )

        # decoding
        lowercase__				 =							tokenizer.decode(_UpperCAmelCase  )
        self.assertEqual(_UpperCAmelCase,				"Unicode €.</s>"  )

        lowercase__				 =							tokenizer("e è é ê ë"  )
        lowercase__				 =							[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
        self.assertEqual(encoded["input_ids"],				_UpperCAmelCase  )
        # decoding
        lowercase__				 =							tokenizer.decode(_UpperCAmelCase  )
        self.assertEqual(_UpperCAmelCase,				"e è é ê ë</s>"  )

        # encode/decode, but with `encode` instead of `__call__`
        self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"  )  ),				"e è é ê ë</s>"  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							self.ta_base_tokenizer
        lowercase__				 =							["A long paragraph for summarization.", "Another paragraph for summarization."]
        # fmt: off
        lowercase__				 =							[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
        # fmt: on
        lowercase__				 =							tokenizer(_UpperCAmelCase,				padding=_UpperCAmelCase,				return_tensors=_UpperCAmelCase  )
        self.assertIsInstance(_UpperCAmelCase,				_UpperCAmelCase  )

        if FRAMEWORK != "jax":
             lowercase__				 =							list(batch.input_ids.numpy()[0]  )
        else:
             lowercase__				 =							list(batch.input_ids.tolist()[0]  )

        self.assertListEqual(_UpperCAmelCase,				_UpperCAmelCase  )

        self.assertEqual((2, 37),				batch.input_ids.shape  )
        self.assertEqual((2, 37),				batch.attention_mask.shape  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							self.ta_base_tokenizer
        lowercase__				 =							["A long paragraph for summarization.", "Another paragraph for summarization."]
        lowercase__				 =							tokenizer(_UpperCAmelCase,				padding=_UpperCAmelCase,				return_tensors=_UpperCAmelCase  )
        # check if input_ids are returned and no decoder_input_ids
        self.assertIn("input_ids",				_UpperCAmelCase  )
        self.assertIn("attention_mask",				_UpperCAmelCase  )
        self.assertNotIn("decoder_input_ids",				_UpperCAmelCase  )
        self.assertNotIn("decoder_attention_mask",				_UpperCAmelCase  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							self.ta_base_tokenizer
        lowercase__				 =							[
            "Summary of the text.",
            "Another summary.",
        ]
        lowercase__				 =							tokenizer(
            text_target=_UpperCAmelCase,				max_length=32,				padding="max_length",				truncation=_UpperCAmelCase,				return_tensors=_UpperCAmelCase  )
        self.assertEqual(32,				targets["input_ids"].shape[1]  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							self.ta_base_tokenizer
        lowercase__				 =							["A long paragraph for summarization. </s>"]
        lowercase__				 =							["Summary of the text. </s>"]
        # fmt: off
        lowercase__				 =							[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
        lowercase__				 =							[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
        # fmt: on

        lowercase__				 =							tokenizer(_UpperCAmelCase,				text_target=_UpperCAmelCase  )

        self.assertEqual(_UpperCAmelCase,				batch["input_ids"][0]  )
        self.assertEqual(_UpperCAmelCase,				batch["labels"][0]  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							self.get_tokenizers()
        for tokenizer in tokenizers:
             with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
                  self.assertNotEqual(tokenizer.model_max_length,				42  )

        # Now let's start the test
        lowercase__				 =							self.get_tokenizers()
        for tokenizer in tokenizers:
             with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
                  # Isolate this from the other tests because we save additional tokens/etc
                  lowercase__				 =							tempfile.mkdtemp()

                  lowercase__				 =							" He is very happy, UNwant\u00E9d,running"
                  lowercase__				 =							tokenizer.encode(_UpperCAmelCase,				add_special_tokens=_UpperCAmelCase  )
                  tokenizer.save_pretrained(_UpperCAmelCase  )

                  lowercase__				 =							tokenizer.__class__.from_pretrained(_UpperCAmelCase  )
                  lowercase__				 =							after_tokenizer.encode(_UpperCAmelCase,				add_special_tokens=_UpperCAmelCase  )
                  self.assertListEqual(_UpperCAmelCase,				_UpperCAmelCase  )

                  shutil.rmtree(_UpperCAmelCase  )

        lowercase__				 =							self.get_tokenizers(model_max_length=42  )
        for tokenizer in tokenizers:
             with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
                  # Isolate this from the other tests because we save additional tokens/etc
                  lowercase__				 =							tempfile.mkdtemp()

                  lowercase__				 =							" He is very happy, UNwant\u00E9d,running"
                  tokenizer.add_tokens(["bim", "bambam"]  )
                  lowercase__				 =							tokenizer.additional_special_tokens
                  additional_special_tokens.append("new_additional_special_token"  )
                  tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}  )
                  lowercase__				 =							tokenizer.encode(_UpperCAmelCase,				add_special_tokens=_UpperCAmelCase  )
                  tokenizer.save_pretrained(_UpperCAmelCase  )

                  lowercase__				 =							tokenizer.__class__.from_pretrained(_UpperCAmelCase  )
                  lowercase__				 =							after_tokenizer.encode(_UpperCAmelCase,				add_special_tokens=_UpperCAmelCase  )
                  self.assertListEqual(_UpperCAmelCase,				_UpperCAmelCase  )
                  self.assertIn("new_additional_special_token",				after_tokenizer.additional_special_tokens  )
                  self.assertEqual(after_tokenizer.model_max_length,				42  )

                  lowercase__				 =							tokenizer.__class__.from_pretrained(_UpperCAmelCase,				model_max_length=43  )
                  self.assertEqual(tokenizer.model_max_length,				43  )

                  shutil.rmtree(_UpperCAmelCase  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							[]
        if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

        if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

        for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                  tokenizer_utils.save_pretrained(_UpperCAmelCase  )

                  with open(os.path.join(_UpperCAmelCase,				"special_tokens_map.json"  ),				encoding="utf-8"  ) as json_file:
                       lowercase__				 =							json.load(_UpperCAmelCase  )

                  with open(os.path.join(_UpperCAmelCase,				"tokenizer_config.json"  ),				encoding="utf-8"  ) as json_file:
                       lowercase__				 =							json.load(_UpperCAmelCase  )

                  lowercase__				 =							[F'''<extra_id_{i}>''' for i in range(125  )]

                  lowercase__				 =							added_tokens_extra_ids + [
                      "an_additional_special_token"
                  ]
                  lowercase__				 =							added_tokens_extra_ids + [
                      "an_additional_special_token"
                  ]

                  with open(os.path.join(_UpperCAmelCase,				"special_tokens_map.json"  ),				"w",				encoding="utf-8"  ) as outfile:
                       json.dump(_UpperCAmelCase,				_UpperCAmelCase  )
                  with open(os.path.join(_UpperCAmelCase,				"tokenizer_config.json"  ),				"w",				encoding="utf-8"  ) as outfile:
                       json.dump(_UpperCAmelCase,				_UpperCAmelCase  )

                  # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                  # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                  # "special_tokens_map.json" files
                  lowercase__				 =							tokenizer_class.from_pretrained(
                      _UpperCAmelCase,				)
                  self.assertIn(
                      "an_additional_special_token",				tokenizer_without_change_in_init.additional_special_tokens  )
                  # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                  self.assertEqual(
                      ["an_additional_special_token"],				tokenizer_without_change_in_init.convert_ids_to_tokens(
                          tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]  )  ),				)

                  # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                  lowercase__				 =							added_tokens_extra_ids + [AddedToken("a_new_additional_special_token",				lstrip=_UpperCAmelCase  )]
                  lowercase__				 =							tokenizer_class.from_pretrained(
                      _UpperCAmelCase,				additional_special_tokens=_UpperCAmelCase,				)

                  self.assertIn("a_new_additional_special_token",				tokenizer.additional_special_tokens  )
                  self.assertEqual(
                      ["a_new_additional_special_token"],				tokenizer.convert_ids_to_tokens(
                          tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]  )  ),				)


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							[]
        if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

        if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

        for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                  tokenizer_utils.save_pretrained(_UpperCAmelCase  )

                  lowercase__				 =							tokenizer_class.from_pretrained(_UpperCAmelCase  )

                  self.assertTrue(tokenizer.decode([255]  ) == ""  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        pass


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        pass


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        pass


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        pass


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							self.get_tokenizers(fast=_UpperCAmelCase,				do_lower_case=_UpperCAmelCase  )
        for tokenizer in tokenizers:
             with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
                  lowercase__				 =							["t", "h", "i", "s", " ", "i", "s", " ", "a", " ", "t", "e", "x", "t", "</s>"]
                  lowercase__				 =							tokenizer.convert_tokens_to_string(_UpperCAmelCase  )

                  self.assertIsInstance(_UpperCAmelCase,				_UpperCAmelCase  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							self.get_tokenizers()
        for tokenizer in tokenizers:
             with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
                  lowercase__				 =							[
                      "bos_token",
                      "eos_token",
                      "unk_token",
                      "sep_token",
                      "pad_token",
                      "cls_token",
                      "mask_token",
                  ]

                  lowercase__				 =							0
                  lowercase__				 =							tokenizer.convert_ids_to_tokens(
                      _UpperCAmelCase,				skip_special_tokens=_UpperCAmelCase  )

                  for attr in attributes_list:
                       setattr(_UpperCAmelCase,				attr + "_id",				_UpperCAmelCase  )
                       self.assertEqual(getattr(_UpperCAmelCase,				_UpperCAmelCase  ),				_UpperCAmelCase  )
                       self.assertEqual(getattr(_UpperCAmelCase,				attr + "_id"  ),				_UpperCAmelCase  )

                       setattr(_UpperCAmelCase,				attr + "_id",				_UpperCAmelCase  )
                       self.assertEqual(getattr(_UpperCAmelCase,				_UpperCAmelCase  ),				_UpperCAmelCase  )
                       self.assertEqual(getattr(_UpperCAmelCase,				attr + "_id"  ),				_UpperCAmelCase  )

                  setattr(_UpperCAmelCase,				"additional_special_tokens_ids",				[]  )
                  self.assertListEqual(getattr(_UpperCAmelCase,				"additional_special_tokens"  ),				[]  )
                  self.assertListEqual(getattr(_UpperCAmelCase,				"additional_special_tokens_ids"  ),				[]  )

                  setattr(_UpperCAmelCase,				"additional_special_tokens_ids",				[token_id_to_test_setters]  )
                  self.assertListEqual(getattr(_UpperCAmelCase,				"additional_special_tokens"  ),				[token_to_test_setters]  )
                  self.assertListEqual(getattr(_UpperCAmelCase,				"additional_special_tokens_ids"  ),				[token_id_to_test_setters]  )




