






import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all image processors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...image_processing_utils import ImageProcessingMixin
from ...utils import CONFIG_NAME, IMAGE_PROCESSOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


UpperCamelCase							       =    logging.get_logger(__name__)

UpperCamelCase							       =    OrderedDict(
    [
        ("""align""", """EfficientNetImageProcessor"""),
        ("""beit""", """BeitImageProcessor"""),
        ("""bit""", """BitImageProcessor"""),
        ("""blip""", """BlipImageProcessor"""),
        ("""blip-2""", """BlipImageProcessor"""),
        ("""bridgetower""", """BridgeTowerImageProcessor"""),
        ("""chinese_clip""", """ChineseCLIPImageProcessor"""),
        ("""clip""", """CLIPImageProcessor"""),
        ("""clipseg""", """ViTImageProcessor"""),
        ("""conditional_detr""", """ConditionalDetrImageProcessor"""),
        ("""convnext""", """ConvNextImageProcessor"""),
        ("""convnextv2""", """ConvNextImageProcessor"""),
        ("""cvt""", """ConvNextImageProcessor"""),
        ("""data2vec-vision""", """BeitImageProcessor"""),
        ("""deformable_detr""", """DeformableDetrImageProcessor"""),
        ("""deit""", """DeiTImageProcessor"""),
        ("""deta""", """DetaImageProcessor"""),
        ("""detr""", """DetrImageProcessor"""),
        ("""dinat""", """ViTImageProcessor"""),
        ("""donut-swin""", """DonutImageProcessor"""),
        ("""dpt""", """DPTImageProcessor"""),
        ("""efficientformer""", """EfficientFormerImageProcessor"""),
        ("""efficientnet""", """EfficientNetImageProcessor"""),
        ("""flava""", """FlavaImageProcessor"""),
        ("""focalnet""", """BitImageProcessor"""),
        ("""git""", """CLIPImageProcessor"""),
        ("""glpn""", """GLPNImageProcessor"""),
        ("""groupvit""", """CLIPImageProcessor"""),
        ("""imagegpt""", """ImageGPTImageProcessor"""),
        ("""instructblip""", """BlipImageProcessor"""),
        ("""layoutlmv2""", """LayoutLMv2ImageProcessor"""),
        ("""layoutlmv3""", """LayoutLMv3ImageProcessor"""),
        ("""levit""", """LevitImageProcessor"""),
        ("""mask2former""", """Mask2FormerImageProcessor"""),
        ("""maskformer""", """MaskFormerImageProcessor"""),
        ("""mgp-str""", """ViTImageProcessor"""),
        ("""mobilenet_v1""", """MobileNetV1ImageProcessor"""),
        ("""mobilenet_v2""", """MobileNetV2ImageProcessor"""),
        ("""mobilevit""", """MobileViTImageProcessor"""),
        ("""mobilevit""", """MobileViTImageProcessor"""),
        ("""mobilevitv2""", """MobileViTImageProcessor"""),
        ("""nat""", """ViTImageProcessor"""),
        ("""oneformer""", """OneFormerImageProcessor"""),
        ("""owlvit""", """OwlViTImageProcessor"""),
        ("""perceiver""", """PerceiverImageProcessor"""),
        ("""pix2struct""", """Pix2StructImageProcessor"""),
        ("""poolformer""", """PoolFormerImageProcessor"""),
        ("""regnet""", """ConvNextImageProcessor"""),
        ("""resnet""", """ConvNextImageProcessor"""),
        ("""sam""", """SamImageProcessor"""),
        ("""segformer""", """SegformerImageProcessor"""),
        ("""swiftformer""", """ViTImageProcessor"""),
        ("""swin""", """ViTImageProcessor"""),
        ("""swin2sr""", """Swin2SRImageProcessor"""),
        ("""swinv2""", """ViTImageProcessor"""),
        ("""table-transformer""", """DetrImageProcessor"""),
        ("""timesformer""", """VideoMAEImageProcessor"""),
        ("""tvlt""", """TvltImageProcessor"""),
        ("""upernet""", """SegformerImageProcessor"""),
        ("""van""", """ConvNextImageProcessor"""),
        ("""videomae""", """VideoMAEImageProcessor"""),
        ("""vilt""", """ViltImageProcessor"""),
        ("""vit""", """ViTImageProcessor"""),
        ("""vit_hybrid""", """ViTHybridImageProcessor"""),
        ("""vit_mae""", """ViTImageProcessor"""),
        ("""vit_msn""", """ViTImageProcessor"""),
        ("""xclip""", """CLIPImageProcessor"""),
        ("""yolos""", """YolosImageProcessor"""),
    ]
)

UpperCamelCase							       =    _LazyAutoMapping(CONFIG_MAPPING_NAMES, IMAGE_PROCESSOR_MAPPING_NAMES)






def        lowerCAmelCase (      UpperCamelCase_:							str					)       ->							Union[str, Any]:







  '''simple docstring'''







  for module_name, extractors in IMAGE_PROCESSOR_MAPPING_NAMES.items():
    if class_name in extractors:
      _a							=	model_type_to_module_name(UpperCamelCase_					)

      _a							=	importlib.import_module(f'''.{module_name}'''     , "transformers.models"					)
      try:
        return getattr(UpperCamelCase_     , UpperCamelCase_					)
      except AttributeError:
        continue

  for _, extractor in IMAGE_PROCESSOR_MAPPING._extra_content.items():
    if getattr(UpperCamelCase_     , "__name__"     , UpperCamelCase_					) == class_name:
      return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
  _a							=	importlib.import_module("transformers"					)
  if hasattr(UpperCamelCase_     , UpperCamelCase_					):
    return getattr(UpperCamelCase_     , UpperCamelCase_					)

  return None






def        lowerCAmelCase (      UpperCamelCase_:							Union[str, os.PathLike]     , UpperCamelCase_:							Optional[Union[str, os.PathLike]] = None     , UpperCamelCase_:							bool = False     , UpperCamelCase_:							bool = False     , UpperCamelCase_:							Optional[Dict[str, str]] = None     , UpperCamelCase_:							Optional[Union[bool, str]] = None     , UpperCamelCase_:							Optional[str] = None     , UpperCamelCase_:							bool = False     , **UpperCamelCase_:							Dict     , )       ->							Optional[int]:







  '''simple docstring'''







  _a							=	get_file_from_repo(
      UpperCamelCase_     , UpperCamelCase_     , cache_dir=UpperCamelCase_     , force_download=UpperCamelCase_     , resume_download=UpperCamelCase_     , proxies=UpperCamelCase_     , use_auth_token=UpperCamelCase_     , revision=UpperCamelCase_     , local_files_only=UpperCamelCase_     , )
  if resolved_config_file is None:
    logger.info(
        "Could not locate the image processor configuration file, will try to use the model config instead."					)
    return {}

  with open(UpperCamelCase_     , encoding="utf-8"					) as reader:
    return json.load(UpperCamelCase_					)




class    lowercase_    :






      def __init__(    self	)	->List[Any]:







        '''simple docstring'''


        raise EnvironmentError(
            "AutoImageProcessor is designed to be instantiated "
            "using the `AutoImageProcessor.from_pretrained(pretrained_model_name_or_path)` method."	)






      @classmethod
      @replace_list_option_in_docstrings(a_	)
      def 			lowerCamelCase__    (    cls     ,     a_     ,     **a_	)	->Dict:







        '''simple docstring'''


        _a							=	kwargs.pop("config"     ,     a_	)
        _a							=	kwargs.pop("trust_remote_code"     ,     a_	)
        _a							=	True

        _a			,	_a							=	ImageProcessingMixin.get_image_processor_dict(a_     ,     **a_	)
        _a							=	config_dict.get("image_processor_type"     ,     a_	)
        _a							=	None
        if "AutoImageProcessor" in config_dict.get("auto_map"     ,     {}	):
          _a							=	config_dict["auto_map"]["AutoImageProcessor"]

        # If we still don't have the image processor class, check if we're loading from a previous feature extractor config
        # and if so, infer the image processor class from there.
        if image_processor_class is None and image_processor_auto_map is None:
          _a							=	config_dict.pop("feature_extractor_type"     ,     a_	)
          if feature_extractor_class is not None:
            logger.warning(
                "Could not find image processor class in the image processor config or the model config. Loading"
                " based on pattern matching with the model's feature extractor configuration."	)
            _a							=	feature_extractor_class.replace("FeatureExtractor"     ,     "ImageProcessor"	)
          if "AutoFeatureExtractor" in config_dict.get("auto_map"     ,     {}	):
            _a							=	config_dict["auto_map"]["AutoFeatureExtractor"]
            _a							=	feature_extractor_auto_map.replace("FeatureExtractor"     ,     "ImageProcessor"	)
            logger.warning(
                "Could not find image processor auto map in the image processor config or the model config."
                " Loading based on pattern matching with the model's feature extractor configuration."	)

        # If we don't find the image processor class in the image processor config, let's try the model config.
        if image_processor_class is None and image_processor_auto_map is None:
          if not isinstance(a_     ,     a_	):
            _a							=	AutoConfig.from_pretrained(a_     ,     **a_	)
          # It could be in `config.image_processor_type``
          _a							=	getattr(a_     ,     "image_processor_type"     ,     a_	)
          if hasattr(a_     ,     "auto_map"	) and "AutoImageProcessor" in config.auto_map:
            _a							=	config.auto_map["AutoImageProcessor"]

        if image_processor_class is not None:
          _a							=	image_processor_class_from_name(a_	)

        _a							=	image_processor_auto_map is not None
        _a							=	image_processor_class is not None or type(a_	) in IMAGE_PROCESSOR_MAPPING
        _a							=	resolve_trust_remote_code(
            a_     ,     a_     ,     a_     ,     a_	)

        if has_remote_code and trust_remote_code:
          _a							=	get_class_from_dynamic_module(
              a_     ,     a_     ,     **a_	)
          _a							=	kwargs.pop("code_revision"     ,     a_	)
          if os.path.isdir(a_	):
            image_processor_class.register_for_auto_class()
          return image_processor_class.from_dict(a_     ,     **a_	)
        elif image_processor_class is not None:
          return image_processor_class.from_dict(a_     ,     **a_	)
        # Last try: we use the IMAGE_PROCESSOR_MAPPING.
        elif type(a_	) in IMAGE_PROCESSOR_MAPPING:
          _a							=	IMAGE_PROCESSOR_MAPPING[type(a_	)]
          return image_processor_class.from_dict(a_     ,     **a_	)

        raise ValueError(
            f'''Unrecognized image processor in {pretrained_model_name_or_path}. Should have a '''
            f'''`image_processor_type` key in its {IMAGE_PROCESSOR_NAME} of {CONFIG_NAME}, or one of the following '''
            f'''`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in IMAGE_PROCESSOR_MAPPING_NAMES.keys()	)}'''	)






      @staticmethod
      def 			lowerCamelCase__    (    a_     ,     a_	)	->Optional[int]:







        '''simple docstring'''


        IMAGE_PROCESSOR_MAPPING.register(a_     ,     a_	)










"""simple docstring"""




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
     UpperCamelCase							       =    """pt"""
elif is_tf_available():
     UpperCamelCase							       =    """tf"""
else:
     UpperCamelCase							       =    """jax"""




class    lowercase_    (_UpperCAmelCase,    unittest.TestCase     ):
      A__	:		List[str]       			=				ByTaTokenizer
      A__	:		List[Any]       			=				False






      def 			lowerCamelCase__    (    self	)	->Any:







        '''simple docstring'''


        super().setUp()
        _a							=	ByTaTokenizer()
        tokenizer.save_pretrained(self.tmpdirname	)






      @cached_property
      def 			lowerCamelCase__    (    self	)	->List[str]:







        '''simple docstring'''


        return ByTaTokenizer.from_pretrained("google/byt5-small"	)






      def 			lowerCamelCase__    (    self     ,     **a_	)	->ByTaTokenizer:







        '''simple docstring'''


        return self.tokenizer_class.from_pretrained(self.tmpdirname     ,     **a_	)






      def 			lowerCamelCase__    (    self     ,     a_     ,     a_=False     ,     a_=2_0     ,     a_=5	)	->Tuple[str, list]:







        '''simple docstring'''


        _a							=	[]
        for i in range(len(a_	)	):
          try:
            _a							=	tokenizer.decode([i]     ,     clean_up_tokenization_spaces=a_	)
          except UnicodeDecodeError:
            pass
          toks.append((i, tok)	)

        _a							=	list(filter(lambda a_	: re.match(R"^[ a-zA-Z]+$"     ,     t[1]	)     ,     a_	)	)
        _a							=	list(filter(lambda a_	: [t[0]] == tokenizer.encode(t[1]     ,     add_special_tokens=a_	)     ,     a_	)	)
        if max_length is not None and len(a_	) > max_length:
          _a							=	toks[:max_length]
        if min_length is not None and len(a_	) < min_length and len(a_	) > 0:
          while len(a_	) < min_length:
            _a							=	toks + toks
        # toks_str = [t[1] for t in toks]
        _a							=	[t[0] for t in toks]

        # Ensure consistency
        _a							=	tokenizer.decode(a_     ,     clean_up_tokenization_spaces=a_	)
        if " " not in output_txt and len(a_	) > 1:
          _a							=	(
              tokenizer.decode([toks_ids[0]]     ,     clean_up_tokenization_spaces=a_	)
              + " "
              + tokenizer.decode(toks_ids[1:]     ,     clean_up_tokenization_spaces=a_	)
          )
        if with_prefix_space:
          _a							=	" " + output_txt
        _a							=	tokenizer.encode(a_     ,     add_special_tokens=a_	)
        return output_txt, output_ids






      def 			lowerCamelCase__    (    self	)	->Union[str, Any]:







        '''simple docstring'''


        _a							=	self.ta_base_tokenizer
        _a							=	tokenizer(["hi</s>", "I went to the gym</s>", "</s>"]	)
        _a							=	tokenizer(["hi", "I went to the gym", ""]	)
        self.assertListEqual(batch_with_eos_added["input_ids"]     ,     batch_without_eos_added["input_ids"]	)






      def 			lowerCamelCase__    (    self	)	->Tuple:







        '''simple docstring'''


        _a							=	self.ta_base_tokenizer
        _a							=	"Unicode €."
        _a							=	tokenizer(a_	)
        _a							=	[8_8, 1_1_3, 1_0_8, 1_0_2, 1_1_4, 1_0_3, 1_0_4, 3_5, 2_2_9, 1_3_3, 1_7_5, 4_9, 1]
        self.assertEqual(encoded["input_ids"]     ,     a_	)

        # decoding
        _a							=	tokenizer.decode(a_	)
        self.assertEqual(a_     ,     "Unicode €.</s>"	)

        _a							=	tokenizer("e è é ê ë"	)
        _a							=	[1_0_4, 3_5, 1_9_8, 1_7_1, 3_5, 1_9_8, 1_7_2, 3_5, 1_9_8, 1_7_3, 3_5, 1_9_8, 1_7_4, 1]
        self.assertEqual(encoded["input_ids"]     ,     a_	)
        # decoding
        _a							=	tokenizer.decode(a_	)
        self.assertEqual(a_     ,     "e è é ê ë</s>"	)

        # encode/decode, but with `encode` instead of `__call__`
        self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"	)	)     ,     "e è é ê ë</s>"	)






      def 			lowerCamelCase__    (    self	)	->int:







        '''simple docstring'''


        _a							=	self.ta_base_tokenizer
        _a							=	["A long paragraph for summarization.", "Another paragraph for summarization."]
        # fmt: off
        _a							=	[6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 1, 0]
        # fmt: on
        _a							=	tokenizer(a_     ,     padding=a_     ,     return_tensors=a_	)
        self.assertIsInstance(a_     ,     a_	)

        if FRAMEWORK != "jax":
          _a							=	list(batch.input_ids.numpy()[0]	)
        else:
          _a							=	list(batch.input_ids.tolist()[0]	)

        self.assertListEqual(a_     ,     a_	)

        self.assertEqual((2, 3_7)     ,     batch.input_ids.shape	)
        self.assertEqual((2, 3_7)     ,     batch.attention_mask.shape	)






      def 			lowerCamelCase__    (    self	)	->Dict:







        '''simple docstring'''


        _a							=	self.ta_base_tokenizer
        _a							=	["A long paragraph for summarization.", "Another paragraph for summarization."]
        _a							=	tokenizer(a_     ,     padding=a_     ,     return_tensors=a_	)
        # check if input_ids are returned and no decoder_input_ids
        self.assertIn("input_ids"     ,     a_	)
        self.assertIn("attention_mask"     ,     a_	)
        self.assertNotIn("decoder_input_ids"     ,     a_	)
        self.assertNotIn("decoder_attention_mask"     ,     a_	)






      def 			lowerCamelCase__    (    self	)	->List[Any]:







        '''simple docstring'''


        _a							=	self.ta_base_tokenizer
        _a							=	[
            "Summary of the text.",
            "Another summary.",
        ]
        _a							=	tokenizer(
            text_target=a_     ,     max_length=3_2     ,     padding="max_length"     ,     truncation=a_     ,     return_tensors=a_	)
        self.assertEqual(3_2     ,     targets["input_ids"].shape[1]	)






      def 			lowerCamelCase__    (    self	)	->Dict:







        '''simple docstring'''


        _a							=	self.ta_base_tokenizer
        _a							=	["A long paragraph for summarization. </s>"]
        _a							=	["Summary of the text. </s>"]
        # fmt: off
        _a							=	[6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 3_5, 1]
        _a							=	[8_6, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_2_4, 3_5, 1_1_4, 1_0_5, 3_5, 1_1_9, 1_0_7, 1_0_4, 3_5, 1_1_9, 1_0_4, 1_2_3, 1_1_9, 4_9, 3_5, 1]
        # fmt: on

        _a							=	tokenizer(a_     ,     text_target=a_	)

        self.assertEqual(a_     ,     batch["input_ids"][0]	)
        self.assertEqual(a_     ,     batch["labels"][0]	)






      def 			lowerCamelCase__    (    self	)	->int:







        '''simple docstring'''


        _a							=	self.get_tokenizers()
        for tokenizer in tokenizers:
          with self.subTest(f'''{tokenizer.__class__.__name__}'''	):
            self.assertNotEqual(tokenizer.model_max_length     ,     4_2	)

        # Now let's start the test
        _a							=	self.get_tokenizers()
        for tokenizer in tokenizers:
          with self.subTest(f'''{tokenizer.__class__.__name__}'''	):
            # Isolate this from the other tests because we save additional tokens/etc
            _a							=	tempfile.mkdtemp()

            _a							=	" He is very happy, UNwant\u00E9d,running"
            _a							=	tokenizer.encode(a_     ,     add_special_tokens=a_	)
            tokenizer.save_pretrained(a_	)

            _a							=	tokenizer.__class__.from_pretrained(a_	)
            _a							=	after_tokenizer.encode(a_     ,     add_special_tokens=a_	)
            self.assertListEqual(a_     ,     a_	)

            shutil.rmtree(a_	)

        _a							=	self.get_tokenizers(model_max_length=4_2	)
        for tokenizer in tokenizers:
          with self.subTest(f'''{tokenizer.__class__.__name__}'''	):
            # Isolate this from the other tests because we save additional tokens/etc
            _a							=	tempfile.mkdtemp()

            _a							=	" He is very happy, UNwant\u00E9d,running"
            tokenizer.add_tokens(["bim", "bambam"]	)
            _a							=	tokenizer.additional_special_tokens
            additional_special_tokens.append("new_additional_special_token"	)
            tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}	)
            _a							=	tokenizer.encode(a_     ,     add_special_tokens=a_	)
            tokenizer.save_pretrained(a_	)

            _a							=	tokenizer.__class__.from_pretrained(a_	)
            _a							=	after_tokenizer.encode(a_     ,     add_special_tokens=a_	)
            self.assertListEqual(a_     ,     a_	)
            self.assertIn("new_additional_special_token"     ,     after_tokenizer.additional_special_tokens	)
            self.assertEqual(after_tokenizer.model_max_length     ,     4_2	)

            _a							=	tokenizer.__class__.from_pretrained(a_     ,     model_max_length=4_3	)
            self.assertEqual(tokenizer.model_max_length     ,     4_3	)

            shutil.rmtree(a_	)






      def 			lowerCamelCase__    (    self	)	->List[str]:







        '''simple docstring'''


        _a							=	[]
        if self.test_slow_tokenizer:
          tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())	)

        if self.test_rust_tokenizer:
          tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())	)

        for tokenizer_class, tokenizer_utils in tokenizer_list:
          with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer_utils.save_pretrained(a_	)

            with open(os.path.join(a_     ,     "special_tokens_map.json"	)     ,     encoding="utf-8"	) as json_file:
              _a							=	json.load(a_	)

            with open(os.path.join(a_     ,     "tokenizer_config.json"	)     ,     encoding="utf-8"	) as json_file:
              _a							=	json.load(a_	)

            _a							=	[f'''<extra_id_{i}>''' for i in range(1_2_5	)]

            _a							=	added_tokens_extra_ids + [
                "an_additional_special_token"
            ]
            _a							=	added_tokens_extra_ids + [
                "an_additional_special_token"
            ]

            with open(os.path.join(a_     ,     "special_tokens_map.json"	)     ,     "w"     ,     encoding="utf-8"	) as outfile:
              json.dump(a_     ,     a_	)
            with open(os.path.join(a_     ,     "tokenizer_config.json"	)     ,     "w"     ,     encoding="utf-8"	) as outfile:
              json.dump(a_     ,     a_	)

            # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
            # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
            # "special_tokens_map.json" files
            _a							=	tokenizer_class.from_pretrained(
                a_     ,     )
            self.assertIn(
                "an_additional_special_token"     ,     tokenizer_without_change_in_init.additional_special_tokens	)
            # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
            self.assertEqual(
                ["an_additional_special_token"]     ,     tokenizer_without_change_in_init.convert_ids_to_tokens(
                    tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]	)	)     ,     )

            # Now we test that we can change the value of additional_special_tokens in the from_pretrained
            _a							=	added_tokens_extra_ids + [AddedToken("a_new_additional_special_token"     ,     lstrip=a_	)]
            _a							=	tokenizer_class.from_pretrained(
                a_     ,     additional_special_tokens=a_     ,     )

            self.assertIn("a_new_additional_special_token"     ,     tokenizer.additional_special_tokens	)
            self.assertEqual(
                ["a_new_additional_special_token"]     ,     tokenizer.convert_ids_to_tokens(
                    tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]	)	)     ,     )






      def 			lowerCamelCase__    (    self	)	->List[str]:







        '''simple docstring'''


        _a							=	[]
        if self.test_slow_tokenizer:
          tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())	)

        if self.test_rust_tokenizer:
          tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())	)

        for tokenizer_class, tokenizer_utils in tokenizer_list:
          with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer_utils.save_pretrained(a_	)

            _a							=	tokenizer_class.from_pretrained(a_	)

            self.assertTrue(tokenizer.decode([2_5_5]	) == ""	)






      def 			lowerCamelCase__    (    self	)	->Optional[int]:







        '''simple docstring'''


        pass






      def 			lowerCamelCase__    (    self	)	->List[Any]:







        '''simple docstring'''


        pass






      def 			lowerCamelCase__    (    self	)	->Tuple:







        '''simple docstring'''


        pass






      def 			lowerCamelCase__    (    self	)	->List[Any]:







        '''simple docstring'''


        pass






      def 			lowerCamelCase__    (    self	)	->List[Any]:







        '''simple docstring'''


        _a							=	self.get_tokenizers(fast=a_     ,     do_lower_case=a_	)
        for tokenizer in tokenizers:
          with self.subTest(f'''{tokenizer.__class__.__name__}'''	):
            _a							=	["t", "h", "i", "s", " ", "i", "s", " ", "a", " ", "t", "e", "x", "t", "</s>"]
            _a							=	tokenizer.convert_tokens_to_string(a_	)

            self.assertIsInstance(a_     ,     a_	)






      def 			lowerCamelCase__    (    self	)	->Union[str, Any]:







        '''simple docstring'''


        _a							=	self.get_tokenizers()
        for tokenizer in tokenizers:
          with self.subTest(f'''{tokenizer.__class__.__name__}'''	):
            _a							=	[
                "bos_token",
                "eos_token",
                "unk_token",
                "sep_token",
                "pad_token",
                "cls_token",
                "mask_token",
            ]

            _a							=	0
            _a							=	tokenizer.convert_ids_to_tokens(
                a_     ,     skip_special_tokens=a_	)

            for attr in attributes_list:
              setattr(a_     ,     attr + "_id"     ,     a_	)
              self.assertEqual(getattr(a_     ,     a_	)     ,     a_	)
              self.assertEqual(getattr(a_     ,     attr + "_id"	)     ,     a_	)

              setattr(a_     ,     attr + "_id"     ,     a_	)
              self.assertEqual(getattr(a_     ,     a_	)     ,     a_	)
              self.assertEqual(getattr(a_     ,     attr + "_id"	)     ,     a_	)

            setattr(a_     ,     "additional_special_tokens_ids"     ,     []	)
            self.assertListEqual(getattr(a_     ,     "additional_special_tokens"	)     ,     []	)
            self.assertListEqual(getattr(a_     ,     "additional_special_tokens_ids"	)     ,     []	)

            setattr(a_     ,     "additional_special_tokens_ids"     ,     [token_id_to_test_setters]	)
            self.assertListEqual(getattr(a_     ,     "additional_special_tokens"	)     ,     [token_to_test_setters]	)
            self.assertListEqual(getattr(a_     ,     "additional_special_tokens_ids"	)     ,     [token_id_to_test_setters]	)


