


"""simple docstring"""





import math
from dataclasses import dataclass
from typing import Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, randn_tensor
from .scheduling_utils import SchedulerMixin



@dataclass
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->UnCLIP
class 		snake_case_(       a__			):
     __UpperCamelCase  =					42
     __UpperCamelCase  =					None



def      _snake_case						(						_snake_case				:			Dict ,   _snake_case				:			List[str]=0.999 ,   _snake_case				:			Dict="cosine" ,   ):
      if alpha_transform_type == "cosine":

            def alpha_bar_fn(_snake_case				:			List[Any]			):
                  return math.cos((t + 0.008) / 1.008 * math.pi / 2			) ** 2

      elif alpha_transform_type == "exp":

            def alpha_bar_fn(_snake_case				:			Optional[int]			):
                  return math.exp(t * -12.0			)

      else:
            raise ValueError(f'''Unsupported alpha_tranform_type: {alpha_transform_type}'''			)

      lowerCAmelCase					:	List[Any]       =   []
      for i in range(_snake_case			):
            lowerCAmelCase					:	int       =   i / num_diffusion_timesteps
            lowerCAmelCase					:	Tuple       =   (i + 1) / num_diffusion_timesteps
            betas.append(min(1 - alpha_bar_fn(_snake_case			) / alpha_bar_fn(_snake_case			) ,   _snake_case			)			)
      return torch.tensor(_snake_case ,   dtype=torch.floataa			)






class 		snake_case_(       a__							,					a__			):






     @register_to_config
     def __init__(	self  :					Any					,				UpperCamelCase_  :					int = 1_0_0_0					,				UpperCamelCase_  :					str = "fixed_small_log"					,				UpperCamelCase_  :					bool = True					,				UpperCamelCase_  :					Optional[float] = 1.0					,				UpperCamelCase_  :					str = "epsilon"					,				UpperCamelCase_  :					str = "squaredcos_cap_v2"					,				):
           if beta_schedule != "squaredcos_cap_v2":
                 raise ValueError('''UnCLIPScheduler only supports `beta_schedule`: \'squaredcos_cap_v2\''''				)

           lowerCAmelCase					:	Any       =   betas_for_alpha_bar(UpperCamelCase_				)

           lowerCAmelCase					:	str       =   1.0 - self.betas
           lowerCAmelCase					:	Union[str, Any]       =   torch.cumprod(self.alphas					,				dim=0				)
           lowerCAmelCase					:	Tuple       =   torch.tensor(1.0				)

           # standard deviation of the initial noise distribution
           lowerCAmelCase					:	Any       =   1.0

           # setable values
           lowerCAmelCase					:	Any       =   None
           lowerCAmelCase					:	Any       =   torch.from_numpy(np.arange(0					,				UpperCamelCase_				)[::-1].copy()				)

           lowerCAmelCase					:	List[str]       =   variance_type






     def      lowerCamelCase__     (	self  :					Union[str, Any]					,				UpperCamelCase_  :					torch.FloatTensor					,				UpperCamelCase_  :					Optional[int] = None				):
           return sample






     def      lowerCamelCase__     (	self  :					Tuple					,				UpperCamelCase_  :					int					,				UpperCamelCase_  :					Union[str, torch.device] = None				):
           lowerCAmelCase					:	Any       =   num_inference_steps
           lowerCAmelCase					:	str       =   (self.config.num_train_timesteps - 1) / (self.num_inference_steps - 1)
           lowerCAmelCase					:	Tuple       =   (np.arange(0					,				UpperCamelCase_				) * step_ratio).round()[::-1].copy().astype(np.intaa				)
           lowerCAmelCase					:	Any       =   torch.from_numpy(UpperCamelCase_				).to(UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Optional[Any]					,				UpperCamelCase_  :					List[Any]					,				UpperCamelCase_  :					str=None					,				UpperCamelCase_  :					Tuple=None					,				UpperCamelCase_  :					Any=None				):
           if prev_timestep is None:
                 lowerCAmelCase					:	Any       =   t - 1

           lowerCAmelCase					:	int       =   self.alphas_cumprod[t]
           lowerCAmelCase					:	Union[str, Any]       =   self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.one
           lowerCAmelCase					:	Dict       =   1 - alpha_prod_t
           lowerCAmelCase					:	str       =   1 - alpha_prod_t_prev

           if prev_timestep == t - 1:
                 lowerCAmelCase					:	Tuple       =   self.betas[t]
           else:
                 lowerCAmelCase					:	Union[str, Any]       =   1 - alpha_prod_t / alpha_prod_t_prev

           # For t > 0, compute predicted variance βt (see formula (6) and (7) from https://arxiv.org/pdf/2006.11239.pdf)
           # and sample from it to get previous sample
           # x_{t-1} ~ N(pred_prev_sample, variance) == add variance to pred_sample
           lowerCAmelCase					:	Optional[Any]       =   beta_prod_t_prev / beta_prod_t * beta

           if variance_type is None:
                 lowerCAmelCase					:	List[str]       =   self.config.variance_type

           # hacks - were probably added for training stability
           if variance_type == "fixed_small_log":
                 lowerCAmelCase					:	Any       =   torch.log(torch.clamp(UpperCamelCase_					,				min=1E-20				)				)
                 lowerCAmelCase					:	Union[str, Any]       =   torch.exp(0.5 * variance				)
           elif variance_type == "learned_range":
                 # NOTE difference with DDPM scheduler
                 lowerCAmelCase					:	Optional[Any]       =   variance.log()
                 lowerCAmelCase					:	Union[str, Any]       =   beta.log()

                 lowerCAmelCase					:	Dict       =   (predicted_variance + 1) / 2
                 lowerCAmelCase					:	Union[str, Any]       =   frac * max_log + (1 - frac) * min_log

           return variance






     def      lowerCamelCase__     (	self  :					Optional[Any]					,				UpperCamelCase_  :					torch.FloatTensor					,				UpperCamelCase_  :					int					,				UpperCamelCase_  :					torch.FloatTensor					,				UpperCamelCase_  :					Optional[int] = None					,				UpperCamelCase_  :					List[Any]=None					,				UpperCamelCase_  :					bool = True					,				):
           lowerCAmelCase					:	Optional[Any]       =   timestep

           if model_output.shape[1] == sample.shape[1] * 2 and self.variance_type == "learned_range":
                 lowerCAmelCase,      lowerCAmelCase					:	List[Any]       =   torch.split(UpperCamelCase_					,				sample.shape[1]					,				dim=1				)
           else:
                 lowerCAmelCase					:	Optional[int]       =   None

           # 1. compute alphas, betas
           if prev_timestep is None:
                 lowerCAmelCase					:	Any       =   t - 1

           lowerCAmelCase					:	Union[str, Any]       =   self.alphas_cumprod[t]
           lowerCAmelCase					:	Optional[int]       =   self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.one
           lowerCAmelCase					:	int       =   1 - alpha_prod_t
           lowerCAmelCase					:	str       =   1 - alpha_prod_t_prev

           if prev_timestep == t - 1:
                 lowerCAmelCase					:	List[Any]       =   self.betas[t]
                 lowerCAmelCase					:	Optional[int]       =   self.alphas[t]
           else:
                 lowerCAmelCase					:	List[Any]       =   1 - alpha_prod_t / alpha_prod_t_prev
                 lowerCAmelCase					:	Dict       =   1 - beta

           # 2. compute predicted original sample from predicted noise also called
           # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf
           if self.config.prediction_type == "epsilon":
                 lowerCAmelCase					:	List[Any]       =   (sample - beta_prod_t ** 0.5 * model_output) / alpha_prod_t ** 0.5
           elif self.config.prediction_type == "sample":
                 lowerCAmelCase					:	Tuple       =   model_output
           else:
                 raise ValueError(
                     F'''prediction_type given as {self.config.prediction_type} must be one of `epsilon` or `sample`'''
                     ''' for the UnCLIPScheduler.'''				)

           # 3. Clip "predicted x_0"
           if self.config.clip_sample:
                 lowerCAmelCase					:	Dict       =   torch.clamp(
                     UpperCamelCase_					,				-self.config.clip_sample_range					,				self.config.clip_sample_range				)

           # 4. Compute coefficients for pred_original_sample x_0 and current sample x_t
           # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf
           lowerCAmelCase					:	int       =   (alpha_prod_t_prev ** 0.5 * beta) / beta_prod_t
           lowerCAmelCase					:	List[Any]       =   alpha ** 0.5 * beta_prod_t_prev / beta_prod_t

           # 5. Compute predicted previous sample µ_t
           # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf
           lowerCAmelCase					:	str       =   pred_original_sample_coeff * pred_original_sample + current_sample_coeff * sample

           # 6. Add noise
           lowerCAmelCase					:	int       =   0
           if t > 0:
                 lowerCAmelCase					:	Union[str, Any]       =   randn_tensor(
                     model_output.shape					,				dtype=model_output.dtype					,				generator=UpperCamelCase_					,				device=model_output.device				)

                 lowerCAmelCase					:	Any       =   self._get_variance(
                     UpperCamelCase_					,				predicted_variance=UpperCamelCase_					,				prev_timestep=UpperCamelCase_					,				)

                 if self.variance_type == "fixed_small_log":
                       lowerCAmelCase					:	str       =   variance
                 elif self.variance_type == "learned_range":
                       lowerCAmelCase					:	Optional[Any]       =   (0.5 * variance).exp()
                 else:
                       raise ValueError(
                           F'''variance_type given as {self.variance_type} must be one of `fixed_small_log` or `learned_range`'''
                           ''' for the UnCLIPScheduler.'''				)

                 lowerCAmelCase					:	List[Any]       =   variance * variance_noise

           lowerCAmelCase					:	int       =   pred_prev_sample + variance

           if not return_dict:
                 return (pred_prev_sample,)

           return UnCLIPSchedulerOutput(prev_sample=UpperCamelCase_					,				pred_original_sample=UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Optional[int]					,				UpperCamelCase_  :					torch.FloatTensor					,				UpperCamelCase_  :					torch.FloatTensor					,				UpperCamelCase_  :					torch.IntTensor					,				):
           # Make sure alphas_cumprod and timestep have same device and dtype as original_samples
           lowerCAmelCase					:	Tuple       =   self.alphas_cumprod.to(device=original_samples.device					,				dtype=original_samples.dtype				)
           lowerCAmelCase					:	int       =   timesteps.to(original_samples.device				)

           lowerCAmelCase					:	Dict       =   alphas_cumprod[timesteps] ** 0.5
           lowerCAmelCase					:	str       =   sqrt_alpha_prod.flatten()
           while len(sqrt_alpha_prod.shape				) < len(original_samples.shape				):
                 lowerCAmelCase					:	Any       =   sqrt_alpha_prod.unsqueeze(-1				)

           lowerCAmelCase					:	List[str]       =   (1 - alphas_cumprod[timesteps]) ** 0.5
           lowerCAmelCase					:	Tuple       =   sqrt_one_minus_alpha_prod.flatten()
           while len(sqrt_one_minus_alpha_prod.shape				) < len(original_samples.shape				):
                 lowerCAmelCase					:	int       =   sqrt_one_minus_alpha_prod.unsqueeze(-1				)

           lowerCAmelCase					:	Dict       =   sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise
           return noisy_samples






"""simple docstring"""





import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
 from test_module.custom_tokenization_fast import CustomTokenizerFast



class 		snake_case_(       unittest.TestCase			):






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           lowerCAmelCase					:	Optional[Any]       =   0






     @slow
     def      lowerCamelCase__     (	self  :					Dict				):
           for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
                 lowerCAmelCase					:	Optional[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
                 self.assertIsNotNone(UpperCamelCase_				)
                 self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)
                 self.assertGreater(len(UpperCamelCase_				)					,				0				)

           for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
                 lowerCAmelCase					:	Tuple       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
                 self.assertIsNotNone(UpperCamelCase_				)
                 self.assertIsInstance(UpperCamelCase_					,				(GPTaTokenizer, GPTaTokenizerFast)				)
                 self.assertGreater(len(UpperCamelCase_				)					,				0				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           lowerCAmelCase					:	Optional[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)
           self.assertEqual(tokenizer.vocab_size					,				1_2				)






     def      lowerCamelCase__     (	self  :					Dict				):
           lowerCAmelCase					:	Tuple       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				(RobertaTokenizer, RobertaTokenizerFast)				)
           self.assertEqual(tokenizer.vocab_size					,				2_0				)






     def      lowerCamelCase__     (	self  :					Dict				):
           lowerCAmelCase					:	int       =   AutoConfig.from_pretrained(UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)
           # Check that tokenizer_type ≠ model_type
           lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				config=UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)
           self.assertEqual(tokenizer.vocab_size					,				1_2				)






     def      lowerCamelCase__     (	self  :					Any				):
           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.txt'''					,				os.path.join(UpperCamelCase_					,				'''vocab.txt'''				)				)

                 lowerCAmelCase					:	Any       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				tokenizer_type='''bert'''					,				use_fast=UpperCamelCase_				)
                 self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.json'''					,				os.path.join(UpperCamelCase_					,				'''vocab.json'''				)				)
                 shutil.copy('''./tests/fixtures/merges.txt'''					,				os.path.join(UpperCamelCase_					,				'''merges.txt'''				)				)

                 lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				tokenizer_type='''gpt2'''					,				use_fast=UpperCamelCase_				)
                 self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.txt'''					,				os.path.join(UpperCamelCase_					,				'''vocab.txt'''				)				)

                 lowerCAmelCase					:	Optional[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				tokenizer_type='''bert'''				)
                 self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.json'''					,				os.path.join(UpperCamelCase_					,				'''vocab.json'''				)				)
                 shutil.copy('''./tests/fixtures/merges.txt'''					,				os.path.join(UpperCamelCase_					,				'''merges.txt'''				)				)

                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				tokenizer_type='''gpt2'''				)
                 self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Dict				):
           with pytest.raises(UpperCamelCase_				):
                 AutoTokenizer.from_pretrained('''./'''					,				tokenizer_type='''xxx'''				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					str				):
           for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                 lowerCAmelCase					:	Dict       =   tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased'''				)
                 self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)

                 if isinstance(UpperCamelCase_					,				UpperCamelCase_				):
                       self.assertEqual(tokenizer.basic_tokenizer.do_lower_case					,				UpperCamelCase_				)
                 else:
                       self.assertEqual(tokenizer.do_lower_case					,				UpperCamelCase_				)

                 self.assertEqual(tokenizer.model_max_length					,				5_1_2				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Optional[int]				):
           for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                 with self.assertRaisesRegex(
                     UpperCamelCase_					,				'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''					,				):
                       lowerCAmelCase					:	Any       =   tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists'''				)






     def      lowerCamelCase__     (	self  :					Tuple				):
           # tests: https://github.com/huggingface/transformers/pull/13251
           # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
           # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
           lowerCAmelCase					:	Optional[Any]       =   TOKENIZER_MAPPING.values()
           lowerCAmelCase					:	Optional[Any]       =   []

           for slow_tok, fast_tok in tokenizers:
                 if slow_tok is not None:
                       tokenizer_names.append(slow_tok.__name__				)

                 if fast_tok is not None:
                       tokenizer_names.append(fast_tok.__name__				)

           for tokenizer_name in tokenizer_names:
                 # must find the right class
                 tokenizer_class_from_name(UpperCamelCase_				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Any				):
           self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''					,				use_fast=UpperCamelCase_				)					,				UpperCamelCase_				)
           self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''				)					,				UpperCamelCase_				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Dict				):
           lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained('''distilbert-base-uncased'''					,				do_lower_case=UpperCamelCase_				)
           lowerCAmelCase					:	Union[str, Any]       =   '''Hello, world. How are you?'''
           lowerCAmelCase					:	Optional[Any]       =   tokenizer.tokenize(UpperCamelCase_				)
           self.assertEqual('''[UNK]'''					,				tokens[0]				)

           lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''					,				do_lower_case=UpperCamelCase_				)
           lowerCAmelCase					:	Optional[int]       =   tokenizer.tokenize(UpperCamelCase_				)
           self.assertEqual('''[UNK]'''					,				tokens[0]				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					int				):
           lowerCAmelCase					:	Union[str, Any]       =   AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config'''				)
           self.assertEqual(type(UpperCamelCase_				)					,				UpperCamelCase_				)
           self.assertEqual(tokenizer.model_max_length					,				5_1_2				)
           self.assertEqual(tokenizer.vocab_size					,				3_0_0_0_0				)
           self.assertEqual(tokenizer.unk_token					,				'''[UNK]'''				)
           self.assertEqual(tokenizer.padding_side					,				'''right'''				)
           self.assertEqual(tokenizer.truncation_side					,				'''right'''				)






     def      lowerCamelCase__     (	self  :					List[Any]				):
           lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(UpperCamelCase_				)
                 lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)

           self.assertIsInstance(UpperCamelCase_					,				tokenizer.__class__				)
           self.assertEqual(tokenizera.vocab_size					,				1_2				)






     def      lowerCamelCase__     (	self  :					List[str]				):
           lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained('''ctrl'''				)
           # There is no fast CTRL so this always gives us a slow tokenizer.
           self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Dict				):
           # Check we can load the tokenizer config of an online model.
           lowerCAmelCase					:	Any       =   get_tokenizer_config('''bert-base-cased'''				)
           lowerCAmelCase					:	Optional[int]       =   config.pop('''_commit_hash'''					,				UpperCamelCase_				)
           # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
           self.assertEqual(UpperCamelCase_					,				{'''do_lower_case''': False}				)

           # This model does not have a tokenizer_config so we get back an empty dict.
           lowerCAmelCase					:	Union[str, Any]       =   get_tokenizer_config(UpperCamelCase_				)
           self.assertDictEqual(UpperCamelCase_					,				{}				)

           # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
           lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(UpperCamelCase_				)
                 lowerCAmelCase					:	Dict       =   get_tokenizer_config(UpperCamelCase_				)

           # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
           self.assertEqual(config['''tokenizer_class''']					,				'''BertTokenizer'''				)






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           try:
                 AutoConfig.register('''custom'''					,				UpperCamelCase_				)

                 AutoTokenizer.register(UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_				)
                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(UpperCamelCase_				):
                       AutoTokenizer.register(UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_				)

                 lowerCAmelCase					:	Union[str, Any]       =   CustomTokenizer.from_pretrained(UpperCamelCase_				)
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer.save_pretrained(UpperCamelCase_				)

                       lowerCAmelCase					:	Tuple       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
                       self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                       del TOKENIZER_MAPPING._extra_content[CustomConfig]






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					str				):
           try:
                 AutoConfig.register('''custom'''					,				UpperCamelCase_				)

                 # Can register in two steps
                 AutoTokenizer.register(UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_				)
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,				(CustomTokenizer, None)				)
                 AutoTokenizer.register(UpperCamelCase_					,				fast_tokenizer_class=UpperCamelCase_				)
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,				(CustomTokenizer, CustomTokenizerFast)				)

                 del TOKENIZER_MAPPING._extra_content[CustomConfig]
                 # Can register in one step
                 AutoTokenizer.register(
                     UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_					,				fast_tokenizer_class=UpperCamelCase_				)
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,				(CustomTokenizer, CustomTokenizerFast)				)

                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(UpperCamelCase_				):
                       AutoTokenizer.register(UpperCamelCase_					,				fast_tokenizer_class=UpperCamelCase_				)

                 # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
                 # and that model does not have a tokenizer.json
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       lowerCAmelCase					:	Dict       =   BertTokenizerFast.from_pretrained(UpperCamelCase_				)
                       bert_tokenizer.save_pretrained(UpperCamelCase_				)
                       lowerCAmelCase					:	int       =   CustomTokenizerFast.from_pretrained(UpperCamelCase_				)

                 with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer.save_pretrained(UpperCamelCase_				)

                       lowerCAmelCase					:	Optional[int]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
                       self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

                       lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                       self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                       del TOKENIZER_MAPPING._extra_content[CustomConfig]






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           # If remote code is not set, we will time out when asking whether to load the model.
           with self.assertRaises(UpperCamelCase_				):
                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''				)
           # If remote code is disabled, we can't load this config.
           with self.assertRaises(UpperCamelCase_				):
                 lowerCAmelCase					:	str       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_				)

           lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_				)
           self.assertTrue(tokenizer.special_attribute_present				)
           # Test tokenizer can be reloaded.
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(UpperCamelCase_				)
                 lowerCAmelCase					:	Optional[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				trust_remote_code=UpperCamelCase_				)
           self.assertTrue(reloaded_tokenizer.special_attribute_present				)

           if is_tokenizers_available():
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)
                 self.assertEqual(reloaded_tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)

                 # Test we can also load the slow version
                 lowerCAmelCase					:	Union[str, Any]       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertTrue(tokenizer.special_attribute_present				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 # Test tokenizer can be reloaded.
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer.save_pretrained(UpperCamelCase_				)
                       lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertEqual(reloaded_tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertTrue(reloaded_tokenizer.special_attribute_present				)
           else:
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertEqual(reloaded_tokenizer.__class__.__name__					,				'''NewTokenizer'''				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Optional[int]				):



           class 		snake_case_(       a__			):
                __UpperCamelCase  =					False



           class 		snake_case_(       a__			):
                __UpperCamelCase  =					NewTokenizer
                __UpperCamelCase  =					False

           try:
                 AutoConfig.register('''custom'''					,				UpperCamelCase_				)
                 AutoTokenizer.register(UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_				)
                 AutoTokenizer.register(UpperCamelCase_					,				fast_tokenizer_class=UpperCamelCase_				)
                 # If remote code is not set, the default is to use local
                 lowerCAmelCase					:	Optional[int]       =   AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)
                 self.assertFalse(tokenizer.special_attribute_present				)
                 lowerCAmelCase					:	str       =   AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''					,				use_fast=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertFalse(tokenizer.special_attribute_present				)

                 # If remote code is disabled, we load the local one.
                 lowerCAmelCase					:	Union[str, Any]       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)
                 self.assertFalse(tokenizer.special_attribute_present				)
                 lowerCAmelCase					:	Dict       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertFalse(tokenizer.special_attribute_present				)

                 # If remote is enabled, we load from the Hub
                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)
                 self.assertTrue(tokenizer.special_attribute_present				)
                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertTrue(tokenizer.special_attribute_present				)

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                       del TOKENIZER_MAPPING._extra_content[CustomConfig]






     def      lowerCamelCase__     (	self  :					Tuple				):
           lowerCAmelCase					:	str       =   AutoTokenizer.from_pretrained(
               '''hf-internal-testing/test_dynamic_tokenizer_legacy'''					,				trust_remote_code=UpperCamelCase_				)
           self.assertTrue(tokenizer.special_attribute_present				)
           if is_tokenizers_available():
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)

                 # Test we can also load the slow version
                 lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer_legacy'''					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertTrue(tokenizer.special_attribute_present				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
           else:
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)






     def      lowerCamelCase__     (	self  :					str				):
           with self.assertRaisesRegex(
               UpperCamelCase_					,				'''bert-base is not a local folder and is not a valid model identifier'''				):
                 lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained('''bert-base'''				)






     def      lowerCamelCase__     (	self  :					int				):
           with self.assertRaisesRegex(
               UpperCamelCase_					,				r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''				):
                 lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				revision='''aaaaaa'''				)






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           # Make sure we have cached the tokenizer.
           lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''				)
           with RequestCounter() as counter:
                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''				)
                 self.assertEqual(counter.get_request_count					,				0				)
                 self.assertEqual(counter.head_request_count					,				1				)
                 self.assertEqual(counter.other_request_count					,				0				)


