






'''simple docstring'''







import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast





class 	SCREAMING_SNAKE_CASE					(					unittest.TestCase       ):






			def 			SCREAMING_SNAKE_CASE      ( self      :			str					)			->			Tuple:
							a_		:  Optional[Any]											=      0






			@slow
			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			List[str]:
							for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
											a_		:  Any											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					)
											self.assertIsNotNone(__SCREAMING_SNAKE_CASE					)
											self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						(BertTokenizer, BertTokenizerFast)					)
											self.assertGreater(len(__SCREAMING_SNAKE_CASE					)  ,						0					)

							for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
											a_		:  Optional[int]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					)
											self.assertIsNotNone(__SCREAMING_SNAKE_CASE					)
											self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						(GPTaTokenizer, GPTaTokenizerFast)					)
											self.assertGreater(len(__SCREAMING_SNAKE_CASE					)  ,						0					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict					)			->			str:
							a_		:  Optional[Any]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					)
							self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						(BertTokenizer, BertTokenizerFast)					)
							self.assertEqual(tokenizer.vocab_size  ,						12					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[Any]					)			->			Union[str, Any]:
							a_		:  List[Any]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					)
							self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						(RobertaTokenizer, RobertaTokenizerFast)					)
							self.assertEqual(tokenizer.vocab_size  ,						20					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			str:
							a_		:  int											=      AutoConfig.from_pretrained(__SCREAMING_SNAKE_CASE					)
							self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)
							# Check that tokenizer_type ≠ model_type
							a_		:  Optional[Any]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE  ,						config=__SCREAMING_SNAKE_CASE					)
							self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						(BertTokenizer, BertTokenizerFast)					)
							self.assertEqual(tokenizer.vocab_size  ,						12					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			Any:
							with tempfile.TemporaryDirectory() as tmp_dir:
											shutil.copy('''./tests/fixtures/vocab.txt'''  ,						os.path.join(__SCREAMING_SNAKE_CASE  ,						'''vocab.txt'''					)					)

											a_		:  Optional[int]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE  ,						tokenizer_type='''bert'''  ,						use_fast=__SCREAMING_SNAKE_CASE					)
											self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							with tempfile.TemporaryDirectory() as tmp_dir:
											shutil.copy('''./tests/fixtures/vocab.json'''  ,						os.path.join(__SCREAMING_SNAKE_CASE  ,						'''vocab.json'''					)					)
											shutil.copy('''./tests/fixtures/merges.txt'''  ,						os.path.join(__SCREAMING_SNAKE_CASE  ,						'''merges.txt'''					)					)

											a_		:  List[Any]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE  ,						tokenizer_type='''gpt2'''  ,						use_fast=__SCREAMING_SNAKE_CASE					)
											self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)






			@require_tokenizers
			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			List[str]:
							with tempfile.TemporaryDirectory() as tmp_dir:
											shutil.copy('''./tests/fixtures/vocab.txt'''  ,						os.path.join(__SCREAMING_SNAKE_CASE  ,						'''vocab.txt'''					)					)

											a_		:  Union[str, Any]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE  ,						tokenizer_type='''bert'''					)
											self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							with tempfile.TemporaryDirectory() as tmp_dir:
											shutil.copy('''./tests/fixtures/vocab.json'''  ,						os.path.join(__SCREAMING_SNAKE_CASE  ,						'''vocab.json'''					)					)
											shutil.copy('''./tests/fixtures/merges.txt'''  ,						os.path.join(__SCREAMING_SNAKE_CASE  ,						'''merges.txt'''					)					)

											a_		:  int											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE  ,						tokenizer_type='''gpt2'''					)
											self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			int:
							with pytest.raises(__SCREAMING_SNAKE_CASE					):
											AutoTokenizer.from_pretrained('''./'''  ,						tokenizer_type='''xxx'''					)






			@require_tokenizers
			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			Optional[Any]:
							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
											a_		:  List[str]											=      tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased'''					)
											self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						(BertTokenizer, BertTokenizerFast)					)

											if isinstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					):
															self.assertEqual(tokenizer.basic_tokenizer.do_lower_case  ,						__SCREAMING_SNAKE_CASE					)
											else:
															self.assertEqual(tokenizer.do_lower_case  ,						__SCREAMING_SNAKE_CASE					)

											self.assertEqual(tokenizer.model_max_length  ,						512					)






			@require_tokenizers
			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[int]					)			->			int:
							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
											with self.assertRaisesRegex(
											    __SCREAMING_SNAKE_CASE  ,						'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''  ,						):
															a_		:  Tuple											=      tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists'''					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			Optional[Any]:
							# tests: https://github.com/huggingface/transformers/pull/13251
							# 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
							# 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
							a_		:  int											=      TOKENIZER_MAPPING.values()
							a_		:  str											=      []

							for slow_tok, fast_tok in tokenizers:
											if slow_tok is not None:
															tokenizer_names.append(slow_tok.__name__					)

											if fast_tok is not None:
															tokenizer_names.append(fast_tok.__name__					)

							for tokenizer_name in tokenizer_names:
											# must find the right class
											tokenizer_class_from_name(__SCREAMING_SNAKE_CASE					)






			@require_tokenizers
			def 			SCREAMING_SNAKE_CASE      ( self      :			str					)			->			int:
							self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''  ,						use_fast=__SCREAMING_SNAKE_CASE					)  ,						__SCREAMING_SNAKE_CASE					)
							self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''					)  ,						__SCREAMING_SNAKE_CASE					)






			@require_tokenizers
			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Union[str, Any]:
							a_		:  Optional[int]											=      AutoTokenizer.from_pretrained('''distilbert-base-uncased'''  ,						do_lower_case=__SCREAMING_SNAKE_CASE					)
							a_		:  str											=      '''Hello, world. How are you?'''
							a_		:  List[Any]											=      tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
							self.assertEqual('''[UNK]'''  ,						tokens[0]					)

							a_		:  List[str]											=      AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''  ,						do_lower_case=__SCREAMING_SNAKE_CASE					)
							a_		:  List[Any]											=      tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
							self.assertEqual('''[UNK]'''  ,						tokens[0]					)






			@require_tokenizers
			def 			SCREAMING_SNAKE_CASE      ( self      :			List[Any]					)			->			List[str]:
							a_		:  List[str]											=      AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config'''					)
							self.assertEqual(type(__SCREAMING_SNAKE_CASE					)  ,						__SCREAMING_SNAKE_CASE					)
							self.assertEqual(tokenizer.model_max_length  ,						512					)
							self.assertEqual(tokenizer.vocab_size  ,						3_0000					)
							self.assertEqual(tokenizer.unk_token  ,						'''[UNK]'''					)
							self.assertEqual(tokenizer.padding_side  ,						'''right'''					)
							self.assertEqual(tokenizer.truncation_side  ,						'''right'''					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]					)			->			int:
							a_		:  Optional[int]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					)
							self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						(BertTokenizer, BertTokenizerFast)					)
							with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE					)
											a_		:  List[Any]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					)

							self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						tokenizer.__class__					)
							self.assertEqual(tokenizera.vocab_size  ,						12					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			str:
							a_		:  Dict											=      AutoTokenizer.from_pretrained('''ctrl'''					)
							# There is no fast CTRL so this always gives us a slow tokenizer.
							self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]					)			->			Optional[Any]:
							# Check we can load the tokenizer config of an online model.
							a_		:  int											=      get_tokenizer_config('''bert-base-cased'''					)
							a_		:  Union[str, Any]											=      config.pop('''_commit_hash'''  ,						__SCREAMING_SNAKE_CASE					)
							# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
							self.assertEqual(__SCREAMING_SNAKE_CASE  ,						{'''do_lower_case''': False}					)

							# This model does not have a tokenizer_config so we get back an empty dict.
							a_		:  Dict											=      get_tokenizer_config(__SCREAMING_SNAKE_CASE					)
							self.assertDictEqual(__SCREAMING_SNAKE_CASE  ,						{}					)

							# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
							a_		:  Optional[Any]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					)
							with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE					)
											a_		:  Dict											=      get_tokenizer_config(__SCREAMING_SNAKE_CASE					)

							# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
							self.assertEqual(config['''tokenizer_class''']  ,						'''BertTokenizer'''					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]					)			->			str:
							try:
											AutoConfig.register('''custom'''  ,						__SCREAMING_SNAKE_CASE					)

											AutoTokenizer.register(__SCREAMING_SNAKE_CASE  ,						slow_tokenizer_class=__SCREAMING_SNAKE_CASE					)
											# Trying to register something existing in the Transformers library will raise an error
											with self.assertRaises(__SCREAMING_SNAKE_CASE					):
															AutoTokenizer.register(__SCREAMING_SNAKE_CASE  ,						slow_tokenizer_class=__SCREAMING_SNAKE_CASE					)

											a_		:  Optional[int]											=      CustomTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					)
											with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE					)

															a_		:  Optional[int]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					)
															self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							finally:
											if "custom" in CONFIG_MAPPING._extra_content:
															del CONFIG_MAPPING._extra_content["custom"]
											if CustomConfig in TOKENIZER_MAPPING._extra_content:
															del TOKENIZER_MAPPING._extra_content[CustomConfig]






			@require_tokenizers
			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Optional[int]:
							try:
											AutoConfig.register('''custom'''  ,						__SCREAMING_SNAKE_CASE					)

											# Can register in two steps
											AutoTokenizer.register(__SCREAMING_SNAKE_CASE  ,						slow_tokenizer_class=__SCREAMING_SNAKE_CASE					)
											self.assertEqual(TOKENIZER_MAPPING[CustomConfig]  ,						(CustomTokenizer, None)					)
											AutoTokenizer.register(__SCREAMING_SNAKE_CASE  ,						fast_tokenizer_class=__SCREAMING_SNAKE_CASE					)
											self.assertEqual(TOKENIZER_MAPPING[CustomConfig]  ,						(CustomTokenizer, CustomTokenizerFast)					)

											del TOKENIZER_MAPPING._extra_content[CustomConfig]
											# Can register in one step
											AutoTokenizer.register(
											    __SCREAMING_SNAKE_CASE  ,						slow_tokenizer_class=__SCREAMING_SNAKE_CASE  ,						fast_tokenizer_class=__SCREAMING_SNAKE_CASE					)
											self.assertEqual(TOKENIZER_MAPPING[CustomConfig]  ,						(CustomTokenizer, CustomTokenizerFast)					)

											# Trying to register something existing in the Transformers library will raise an error
											with self.assertRaises(__SCREAMING_SNAKE_CASE					):
															AutoTokenizer.register(__SCREAMING_SNAKE_CASE  ,						fast_tokenizer_class=__SCREAMING_SNAKE_CASE					)

											# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
											# and that model does not have a tokenizer.json
											with tempfile.TemporaryDirectory() as tmp_dir:
															a_		:  Optional[int]											=      BertTokenizerFast.from_pretrained(__SCREAMING_SNAKE_CASE					)
															bert_tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE					)
															a_		:  Tuple											=      CustomTokenizerFast.from_pretrained(__SCREAMING_SNAKE_CASE					)

											with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE					)

															a_		:  Union[str, Any]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					)
															self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

															a_		:  Tuple											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE					)
															self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							finally:
											if "custom" in CONFIG_MAPPING._extra_content:
															del CONFIG_MAPPING._extra_content["custom"]
											if CustomConfig in TOKENIZER_MAPPING._extra_content:
															del TOKENIZER_MAPPING._extra_content[CustomConfig]






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[int]					)			->			int:
							# If remote code is not set, we will time out when asking whether to load the model.
							with self.assertRaises(__SCREAMING_SNAKE_CASE					):
											a_		:  Tuple											=      AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''					)
							# If remote code is disabled, we can't load this config.
							with self.assertRaises(__SCREAMING_SNAKE_CASE					):
											a_		:  Optional[int]											=      AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer'''  ,						trust_remote_code=__SCREAMING_SNAKE_CASE					)

							a_		:  List[Any]											=      AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''  ,						trust_remote_code=__SCREAMING_SNAKE_CASE					)
							self.assertTrue(tokenizer.special_attribute_present					)
							# Test tokenizer can be reloaded.
							with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE					)
											a_		:  str											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE  ,						trust_remote_code=__SCREAMING_SNAKE_CASE					)
							self.assertTrue(reloaded_tokenizer.special_attribute_present					)

							if is_tokenizers_available():
											self.assertEqual(tokenizer.__class__.__name__  ,						'''NewTokenizerFast'''					)
											self.assertEqual(reloaded_tokenizer.__class__.__name__  ,						'''NewTokenizerFast'''					)

											# Test we can also load the slow version
											a_		:  Dict											=      AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer'''  ,						trust_remote_code=__SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE					)
											self.assertTrue(tokenizer.special_attribute_present					)
											self.assertEqual(tokenizer.__class__.__name__  ,						'''NewTokenizer'''					)
											# Test tokenizer can be reloaded.
											with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE					)
															a_		:  List[str]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE  ,						trust_remote_code=__SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE					)
											self.assertEqual(reloaded_tokenizer.__class__.__name__  ,						'''NewTokenizer'''					)
											self.assertTrue(reloaded_tokenizer.special_attribute_present					)
							else:
											self.assertEqual(tokenizer.__class__.__name__  ,						'''NewTokenizer'''					)
											self.assertEqual(reloaded_tokenizer.__class__.__name__  ,						'''NewTokenizer'''					)






			@require_tokenizers
			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]					)			->			Optional[int]:





							class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
										snake_case__  =				False





							class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
										snake_case__  =				NewTokenizer
										snake_case__  =				False

							try:
											AutoConfig.register('''custom'''  ,						__SCREAMING_SNAKE_CASE					)
											AutoTokenizer.register(__SCREAMING_SNAKE_CASE  ,						slow_tokenizer_class=__SCREAMING_SNAKE_CASE					)
											AutoTokenizer.register(__SCREAMING_SNAKE_CASE  ,						fast_tokenizer_class=__SCREAMING_SNAKE_CASE					)
											# If remote code is not set, the default is to use local
											a_		:  List[Any]											=      AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''					)
											self.assertEqual(tokenizer.__class__.__name__  ,						'''NewTokenizerFast'''					)
											self.assertFalse(tokenizer.special_attribute_present					)
											a_		:  Dict											=      AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''  ,						use_fast=__SCREAMING_SNAKE_CASE					)
											self.assertEqual(tokenizer.__class__.__name__  ,						'''NewTokenizer'''					)
											self.assertFalse(tokenizer.special_attribute_present					)

											# If remote code is disabled, we load the local one.
											a_		:  Optional[int]											=      AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer'''  ,						trust_remote_code=__SCREAMING_SNAKE_CASE					)
											self.assertEqual(tokenizer.__class__.__name__  ,						'''NewTokenizerFast'''					)
											self.assertFalse(tokenizer.special_attribute_present					)
											a_		:  Any											=      AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer'''  ,						trust_remote_code=__SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE					)
											self.assertEqual(tokenizer.__class__.__name__  ,						'''NewTokenizer'''					)
											self.assertFalse(tokenizer.special_attribute_present					)

											# If remote is enabled, we load from the Hub
											a_		:  str											=      AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer'''  ,						trust_remote_code=__SCREAMING_SNAKE_CASE					)
											self.assertEqual(tokenizer.__class__.__name__  ,						'''NewTokenizerFast'''					)
											self.assertTrue(tokenizer.special_attribute_present					)
											a_		:  Optional[Any]											=      AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer'''  ,						trust_remote_code=__SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE					)
											self.assertEqual(tokenizer.__class__.__name__  ,						'''NewTokenizer'''					)
											self.assertTrue(tokenizer.special_attribute_present					)

							finally:
											if "custom" in CONFIG_MAPPING._extra_content:
															del CONFIG_MAPPING._extra_content["custom"]
											if CustomConfig in TOKENIZER_MAPPING._extra_content:
															del TOKENIZER_MAPPING._extra_content[CustomConfig]






			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			List[str]:
							a_		:  List[Any]											=      AutoTokenizer.from_pretrained(
							    '''hf-internal-testing/test_dynamic_tokenizer_legacy'''  ,						trust_remote_code=__SCREAMING_SNAKE_CASE					)
							self.assertTrue(tokenizer.special_attribute_present					)
							if is_tokenizers_available():
											self.assertEqual(tokenizer.__class__.__name__  ,						'''NewTokenizerFast'''					)

											# Test we can also load the slow version
											a_		:  Tuple											=      AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer_legacy'''  ,						trust_remote_code=__SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE					)
											self.assertTrue(tokenizer.special_attribute_present					)
											self.assertEqual(tokenizer.__class__.__name__  ,						'''NewTokenizer'''					)
							else:
											self.assertEqual(tokenizer.__class__.__name__  ,						'''NewTokenizer'''					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			List[str]:
							with self.assertRaisesRegex(
							    __SCREAMING_SNAKE_CASE  ,						'''bert-base is not a local folder and is not a valid model identifier'''					):
											a_		:  Optional[Any]											=      AutoTokenizer.from_pretrained('''bert-base'''					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[Any]					)			->			Tuple:
							with self.assertRaisesRegex(
							    __SCREAMING_SNAKE_CASE  ,						r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''					):
											a_		:  Optional[int]											=      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE  ,						revision='''aaaaaa'''					)







			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			List[str]:
							# Make sure we have cached the tokenizer.
							a_		:  str											=      AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''					)
							with RequestCounter() as counter:
											a_		:  str											=      AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''					)
											self.assertEqual(counter.get_request_count  ,						0					)
											self.assertEqual(counter.head_request_count  ,						1					)
											self.assertEqual(counter.other_request_count  ,						0					)












'''simple docstring'''







import requests
from bsa import BeautifulSoup





def 		_UpperCAmelCase   (							__A							:   str						,		__A							:   dict      ):
				a_		:  Tuple											=      BeautifulSoup(requests.get(__A						,		params=__A      ).content						,		'''html.parser'''      )
				a_		:  List[str]											=      soup.find('''div'''						,		attrs={'''class''': '''gs_ri'''}      )
				a_		:  List[str]											=      div.find('''div'''						,		attrs={'''class''': '''gs_fl'''}      ).find_all('''a'''      )
				return anchors[2].get_text()


if __name__ == "__main__":
		__lowerCAmelCase       		=      {
		    'title': (
		        'Precisely geometry controlled microsupercapacitors for ultrahigh areal '
		        'capacitance, volumetric capacitance, and energy density'
		    ),
		    'journal': 'Chem. Mater.',
		    'volume': 30,
		    'pages': '3979-3990',
		    'year': 2_018,
		    'hl': 'en',
		}
		print(get_citation('https://scholar.google.com/scholar_lookup', params=params))




