


"""simple docstring"""





import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader


def      _snake_case						(						_snake_case				:			Features			):
      lowerCAmelCase					:	int       =   np.inf

      def set_batch_size(_snake_case				:			FeatureType			) -> None:
            nonlocal batch_size
            if isinstance(_snake_case ,   _snake_case			):
                  lowerCAmelCase					:	int       =   min(_snake_case ,   config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS			)
            elif isinstance(_snake_case ,   _snake_case			):
                  lowerCAmelCase					:	str       =   min(_snake_case ,   config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS			)
            elif isinstance(_snake_case ,   _snake_case			) and feature.dtype == "binary":
                  lowerCAmelCase					:	Dict       =   min(_snake_case ,   config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS			)

      _visit(_snake_case ,   _snake_case			)

      return None if batch_size is np.inf else batch_size



class 		snake_case_(       a__			):






     def __init__(	self  :					int					,				UpperCamelCase_  :					NestedDataStructureLike[PathLike]					,				UpperCamelCase_  :					Optional[NamedSplit] = None					,				UpperCamelCase_  :					Optional[Features] = None					,				UpperCamelCase_  :					str = None					,				UpperCamelCase_  :					bool = False					,				UpperCamelCase_  :					bool = False					,				UpperCamelCase_  :					Optional[int] = None					,				**UpperCamelCase_  :					List[str]					,				):
           super().__init__(
               UpperCamelCase_					,				split=UpperCamelCase_					,				features=UpperCamelCase_					,				cache_dir=UpperCamelCase_					,				keep_in_memory=UpperCamelCase_					,				streaming=UpperCamelCase_					,				num_proc=UpperCamelCase_					,				**UpperCamelCase_					,				)
           lowerCAmelCase					:	Optional[int]       =   path_or_paths if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else {self.split: path_or_paths}
           lowerCAmelCase					:	Optional[Any]       =   _PACKAGED_DATASETS_MODULES['''parquet'''][1]
           lowerCAmelCase					:	Optional[Any]       =   Parquet(
               cache_dir=UpperCamelCase_					,				data_files=UpperCamelCase_					,				features=UpperCamelCase_					,				hash=UpperCamelCase_					,				**UpperCamelCase_					,				)






     def      lowerCamelCase__     (	self  :					Optional[Any]				):
           # Build iterable dataset
           if self.streaming:
                 lowerCAmelCase					:	Union[str, Any]       =   self.builder.as_streaming_dataset(split=self.split				)
           # Build regular (map-style) dataset
           else:
                 lowerCAmelCase					:	Optional[Any]       =   None
                 lowerCAmelCase					:	List[Any]       =   None
                 lowerCAmelCase					:	Union[str, Any]       =   None
                 lowerCAmelCase					:	str       =   None

                 self.builder.download_and_prepare(
                     download_config=UpperCamelCase_					,				download_mode=UpperCamelCase_					,				verification_mode=UpperCamelCase_					,				base_path=UpperCamelCase_					,				num_proc=self.num_proc					,				)
                 lowerCAmelCase					:	Dict       =   self.builder.as_dataset(
                     split=self.split					,				verification_mode=UpperCamelCase_					,				in_memory=self.keep_in_memory				)
           return dataset



class 		snake_case_:






     def __init__(	self  :					Optional[int]					,				UpperCamelCase_  :					Dataset					,				UpperCamelCase_  :					Union[PathLike, BinaryIO]					,				UpperCamelCase_  :					Optional[int] = None					,				**UpperCamelCase_  :					Union[str, Any]					,				):
           lowerCAmelCase					:	Optional[int]       =   dataset
           lowerCAmelCase					:	str       =   path_or_buf
           lowerCAmelCase					:	Optional[int]       =   batch_size or get_writer_batch_size(dataset.features				)
           lowerCAmelCase					:	Optional[Any]       =   parquet_writer_kwargs






     def      lowerCamelCase__     (	self  :					Dict				):
           lowerCAmelCase					:	Optional[Any]       =   self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

           if isinstance(self.path_or_buf					,				(str, bytes, os.PathLike)				):
                 with open(self.path_or_buf					,				'''wb+'''				) as buffer:
                       lowerCAmelCase					:	Any       =   self._write(file_obj=UpperCamelCase_					,				batch_size=UpperCamelCase_					,				**self.parquet_writer_kwargs				)
           else:
                 lowerCAmelCase					:	List[str]       =   self._write(file_obj=self.path_or_buf					,				batch_size=UpperCamelCase_					,				**self.parquet_writer_kwargs				)
           return written






     def      lowerCamelCase__     (	self  :					Optional[Any]					,				UpperCamelCase_  :					BinaryIO					,				UpperCamelCase_  :					int					,				**UpperCamelCase_  :					Dict				):
           lowerCAmelCase					:	str       =   0
           lowerCAmelCase					:	int       =   parquet_writer_kwargs.pop('''path_or_buf'''					,				UpperCamelCase_				)
           lowerCAmelCase					:	List[str]       =   self.dataset.features.arrow_schema

           lowerCAmelCase					:	Tuple       =   pq.ParquetWriter(UpperCamelCase_					,				schema=UpperCamelCase_					,				**UpperCamelCase_				)

           for offset in logging.tqdm(
               range(0					,				len(self.dataset				)					,				UpperCamelCase_				)					,				unit='''ba'''					,				disable=not logging.is_progress_bar_enabled()					,				desc='''Creating parquet from Arrow format'''					,				):
                 lowerCAmelCase					:	Union[str, Any]       =   query_table(
                     table=self.dataset._data					,				key=slice(UpperCamelCase_					,				offset + batch_size				)					,				indices=self.dataset._indices if self.dataset._indices is not None else None					,				)
                 writer.write_table(UpperCamelCase_				)
                 written += batch.nbytes
           writer.close()
           return written






"""simple docstring"""





import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
 from test_module.custom_tokenization_fast import CustomTokenizerFast



class 		snake_case_(       unittest.TestCase			):






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           lowerCAmelCase					:	Optional[Any]       =   0






     @slow
     def      lowerCamelCase__     (	self  :					Dict				):
           for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
                 lowerCAmelCase					:	Optional[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
                 self.assertIsNotNone(UpperCamelCase_				)
                 self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)
                 self.assertGreater(len(UpperCamelCase_				)					,				0				)

           for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
                 lowerCAmelCase					:	Tuple       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
                 self.assertIsNotNone(UpperCamelCase_				)
                 self.assertIsInstance(UpperCamelCase_					,				(GPTaTokenizer, GPTaTokenizerFast)				)
                 self.assertGreater(len(UpperCamelCase_				)					,				0				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           lowerCAmelCase					:	Optional[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)
           self.assertEqual(tokenizer.vocab_size					,				1_2				)






     def      lowerCamelCase__     (	self  :					Dict				):
           lowerCAmelCase					:	Tuple       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				(RobertaTokenizer, RobertaTokenizerFast)				)
           self.assertEqual(tokenizer.vocab_size					,				2_0				)






     def      lowerCamelCase__     (	self  :					Dict				):
           lowerCAmelCase					:	int       =   AutoConfig.from_pretrained(UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)
           # Check that tokenizer_type ≠ model_type
           lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				config=UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)
           self.assertEqual(tokenizer.vocab_size					,				1_2				)






     def      lowerCamelCase__     (	self  :					Any				):
           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.txt'''					,				os.path.join(UpperCamelCase_					,				'''vocab.txt'''				)				)

                 lowerCAmelCase					:	Any       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				tokenizer_type='''bert'''					,				use_fast=UpperCamelCase_				)
                 self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.json'''					,				os.path.join(UpperCamelCase_					,				'''vocab.json'''				)				)
                 shutil.copy('''./tests/fixtures/merges.txt'''					,				os.path.join(UpperCamelCase_					,				'''merges.txt'''				)				)

                 lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				tokenizer_type='''gpt2'''					,				use_fast=UpperCamelCase_				)
                 self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.txt'''					,				os.path.join(UpperCamelCase_					,				'''vocab.txt'''				)				)

                 lowerCAmelCase					:	Optional[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				tokenizer_type='''bert'''				)
                 self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.json'''					,				os.path.join(UpperCamelCase_					,				'''vocab.json'''				)				)
                 shutil.copy('''./tests/fixtures/merges.txt'''					,				os.path.join(UpperCamelCase_					,				'''merges.txt'''				)				)

                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				tokenizer_type='''gpt2'''				)
                 self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Dict				):
           with pytest.raises(UpperCamelCase_				):
                 AutoTokenizer.from_pretrained('''./'''					,				tokenizer_type='''xxx'''				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					str				):
           for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                 lowerCAmelCase					:	Dict       =   tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased'''				)
                 self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)

                 if isinstance(UpperCamelCase_					,				UpperCamelCase_				):
                       self.assertEqual(tokenizer.basic_tokenizer.do_lower_case					,				UpperCamelCase_				)
                 else:
                       self.assertEqual(tokenizer.do_lower_case					,				UpperCamelCase_				)

                 self.assertEqual(tokenizer.model_max_length					,				5_1_2				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Optional[int]				):
           for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                 with self.assertRaisesRegex(
                     UpperCamelCase_					,				'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''					,				):
                       lowerCAmelCase					:	Any       =   tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists'''				)






     def      lowerCamelCase__     (	self  :					Tuple				):
           # tests: https://github.com/huggingface/transformers/pull/13251
           # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
           # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
           lowerCAmelCase					:	Optional[Any]       =   TOKENIZER_MAPPING.values()
           lowerCAmelCase					:	Optional[Any]       =   []

           for slow_tok, fast_tok in tokenizers:
                 if slow_tok is not None:
                       tokenizer_names.append(slow_tok.__name__				)

                 if fast_tok is not None:
                       tokenizer_names.append(fast_tok.__name__				)

           for tokenizer_name in tokenizer_names:
                 # must find the right class
                 tokenizer_class_from_name(UpperCamelCase_				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Any				):
           self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''					,				use_fast=UpperCamelCase_				)					,				UpperCamelCase_				)
           self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''				)					,				UpperCamelCase_				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Dict				):
           lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained('''distilbert-base-uncased'''					,				do_lower_case=UpperCamelCase_				)
           lowerCAmelCase					:	Union[str, Any]       =   '''Hello, world. How are you?'''
           lowerCAmelCase					:	Optional[Any]       =   tokenizer.tokenize(UpperCamelCase_				)
           self.assertEqual('''[UNK]'''					,				tokens[0]				)

           lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''					,				do_lower_case=UpperCamelCase_				)
           lowerCAmelCase					:	Optional[int]       =   tokenizer.tokenize(UpperCamelCase_				)
           self.assertEqual('''[UNK]'''					,				tokens[0]				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					int				):
           lowerCAmelCase					:	Union[str, Any]       =   AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config'''				)
           self.assertEqual(type(UpperCamelCase_				)					,				UpperCamelCase_				)
           self.assertEqual(tokenizer.model_max_length					,				5_1_2				)
           self.assertEqual(tokenizer.vocab_size					,				3_0_0_0_0				)
           self.assertEqual(tokenizer.unk_token					,				'''[UNK]'''				)
           self.assertEqual(tokenizer.padding_side					,				'''right'''				)
           self.assertEqual(tokenizer.truncation_side					,				'''right'''				)






     def      lowerCamelCase__     (	self  :					List[Any]				):
           lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(UpperCamelCase_				)
                 lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)

           self.assertIsInstance(UpperCamelCase_					,				tokenizer.__class__				)
           self.assertEqual(tokenizera.vocab_size					,				1_2				)






     def      lowerCamelCase__     (	self  :					List[str]				):
           lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained('''ctrl'''				)
           # There is no fast CTRL so this always gives us a slow tokenizer.
           self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Dict				):
           # Check we can load the tokenizer config of an online model.
           lowerCAmelCase					:	Any       =   get_tokenizer_config('''bert-base-cased'''				)
           lowerCAmelCase					:	Optional[int]       =   config.pop('''_commit_hash'''					,				UpperCamelCase_				)
           # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
           self.assertEqual(UpperCamelCase_					,				{'''do_lower_case''': False}				)

           # This model does not have a tokenizer_config so we get back an empty dict.
           lowerCAmelCase					:	Union[str, Any]       =   get_tokenizer_config(UpperCamelCase_				)
           self.assertDictEqual(UpperCamelCase_					,				{}				)

           # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
           lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(UpperCamelCase_				)
                 lowerCAmelCase					:	Dict       =   get_tokenizer_config(UpperCamelCase_				)

           # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
           self.assertEqual(config['''tokenizer_class''']					,				'''BertTokenizer'''				)






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           try:
                 AutoConfig.register('''custom'''					,				UpperCamelCase_				)

                 AutoTokenizer.register(UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_				)
                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(UpperCamelCase_				):
                       AutoTokenizer.register(UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_				)

                 lowerCAmelCase					:	Union[str, Any]       =   CustomTokenizer.from_pretrained(UpperCamelCase_				)
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer.save_pretrained(UpperCamelCase_				)

                       lowerCAmelCase					:	Tuple       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
                       self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                       del TOKENIZER_MAPPING._extra_content[CustomConfig]






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					str				):
           try:
                 AutoConfig.register('''custom'''					,				UpperCamelCase_				)

                 # Can register in two steps
                 AutoTokenizer.register(UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_				)
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,				(CustomTokenizer, None)				)
                 AutoTokenizer.register(UpperCamelCase_					,				fast_tokenizer_class=UpperCamelCase_				)
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,				(CustomTokenizer, CustomTokenizerFast)				)

                 del TOKENIZER_MAPPING._extra_content[CustomConfig]
                 # Can register in one step
                 AutoTokenizer.register(
                     UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_					,				fast_tokenizer_class=UpperCamelCase_				)
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,				(CustomTokenizer, CustomTokenizerFast)				)

                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(UpperCamelCase_				):
                       AutoTokenizer.register(UpperCamelCase_					,				fast_tokenizer_class=UpperCamelCase_				)

                 # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
                 # and that model does not have a tokenizer.json
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       lowerCAmelCase					:	Dict       =   BertTokenizerFast.from_pretrained(UpperCamelCase_				)
                       bert_tokenizer.save_pretrained(UpperCamelCase_				)
                       lowerCAmelCase					:	int       =   CustomTokenizerFast.from_pretrained(UpperCamelCase_				)

                 with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer.save_pretrained(UpperCamelCase_				)

                       lowerCAmelCase					:	Optional[int]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
                       self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

                       lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                       self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                       del TOKENIZER_MAPPING._extra_content[CustomConfig]






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           # If remote code is not set, we will time out when asking whether to load the model.
           with self.assertRaises(UpperCamelCase_				):
                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''				)
           # If remote code is disabled, we can't load this config.
           with self.assertRaises(UpperCamelCase_				):
                 lowerCAmelCase					:	str       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_				)

           lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_				)
           self.assertTrue(tokenizer.special_attribute_present				)
           # Test tokenizer can be reloaded.
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(UpperCamelCase_				)
                 lowerCAmelCase					:	Optional[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				trust_remote_code=UpperCamelCase_				)
           self.assertTrue(reloaded_tokenizer.special_attribute_present				)

           if is_tokenizers_available():
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)
                 self.assertEqual(reloaded_tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)

                 # Test we can also load the slow version
                 lowerCAmelCase					:	Union[str, Any]       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertTrue(tokenizer.special_attribute_present				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 # Test tokenizer can be reloaded.
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer.save_pretrained(UpperCamelCase_				)
                       lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertEqual(reloaded_tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertTrue(reloaded_tokenizer.special_attribute_present				)
           else:
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertEqual(reloaded_tokenizer.__class__.__name__					,				'''NewTokenizer'''				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Optional[int]				):



           class 		snake_case_(       a__			):
                __UpperCamelCase  =					False



           class 		snake_case_(       a__			):
                __UpperCamelCase  =					NewTokenizer
                __UpperCamelCase  =					False

           try:
                 AutoConfig.register('''custom'''					,				UpperCamelCase_				)
                 AutoTokenizer.register(UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_				)
                 AutoTokenizer.register(UpperCamelCase_					,				fast_tokenizer_class=UpperCamelCase_				)
                 # If remote code is not set, the default is to use local
                 lowerCAmelCase					:	Optional[int]       =   AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)
                 self.assertFalse(tokenizer.special_attribute_present				)
                 lowerCAmelCase					:	str       =   AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''					,				use_fast=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertFalse(tokenizer.special_attribute_present				)

                 # If remote code is disabled, we load the local one.
                 lowerCAmelCase					:	Union[str, Any]       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)
                 self.assertFalse(tokenizer.special_attribute_present				)
                 lowerCAmelCase					:	Dict       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertFalse(tokenizer.special_attribute_present				)

                 # If remote is enabled, we load from the Hub
                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)
                 self.assertTrue(tokenizer.special_attribute_present				)
                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertTrue(tokenizer.special_attribute_present				)

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                       del TOKENIZER_MAPPING._extra_content[CustomConfig]






     def      lowerCamelCase__     (	self  :					Tuple				):
           lowerCAmelCase					:	str       =   AutoTokenizer.from_pretrained(
               '''hf-internal-testing/test_dynamic_tokenizer_legacy'''					,				trust_remote_code=UpperCamelCase_				)
           self.assertTrue(tokenizer.special_attribute_present				)
           if is_tokenizers_available():
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)

                 # Test we can also load the slow version
                 lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer_legacy'''					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertTrue(tokenizer.special_attribute_present				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
           else:
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)






     def      lowerCamelCase__     (	self  :					str				):
           with self.assertRaisesRegex(
               UpperCamelCase_					,				'''bert-base is not a local folder and is not a valid model identifier'''				):
                 lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained('''bert-base'''				)






     def      lowerCamelCase__     (	self  :					int				):
           with self.assertRaisesRegex(
               UpperCamelCase_					,				r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''				):
                 lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				revision='''aaaaaa'''				)






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           # Make sure we have cached the tokenizer.
           lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''				)
           with RequestCounter() as counter:
                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''				)
                 self.assertEqual(counter.get_request_count					,				0				)
                 self.assertEqual(counter.head_request_count					,				1				)
                 self.assertEqual(counter.other_request_count					,				0				)


