
"""simple docstring"""




import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast





class  snake_case     (							unittest.TestCase   ):

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = 0

							@slow
							def _lowercase					(self    ):

									"""simple docstring"""







									for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase    )
											self.assertIsNotNone(__lowerCamelCase    )
											self.assertIsInstance(__lowerCamelCase       ,       (BertTokenizer, BertTokenizerFast)    )
											self.assertGreater(len(__lowerCamelCase    )       ,       0    )

									for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase    )
											self.assertIsNotNone(__lowerCamelCase    )
											self.assertIsInstance(__lowerCamelCase       ,       (GPTaTokenizer, GPTaTokenizerFast)    )
											self.assertGreater(len(__lowerCamelCase    )       ,       0    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase    )
									self.assertIsInstance(__lowerCamelCase       ,       (BertTokenizer, BertTokenizerFast)    )
									self.assertEqual(tokenizer.vocab_size       ,       12    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase    )
									self.assertIsInstance(__lowerCamelCase       ,       (RobertaTokenizer, RobertaTokenizerFast)    )
									self.assertEqual(tokenizer.vocab_size       ,       20    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = AutoConfig.from_pretrained(__lowerCamelCase    )
									self.assertIsInstance(__lowerCamelCase       ,       __lowerCamelCase    )
									# Check that tokenizer_type ≠ model_type
									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase       ,       config=__lowerCamelCase    )
									self.assertIsInstance(__lowerCamelCase       ,       (BertTokenizer, BertTokenizerFast)    )
									self.assertEqual(tokenizer.vocab_size       ,       12    )

							def _lowercase					(self    ):

									"""simple docstring"""







									with tempfile.TemporaryDirectory() as tmp_dir:
											shutil.copy('''./tests/fixtures/vocab.txt'''       ,       os.path.join(__lowerCamelCase       ,       '''vocab.txt'''    )    )

											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase       ,       tokenizer_type='''bert'''       ,       use_fast=__lowerCamelCase    )
											self.assertIsInstance(__lowerCamelCase       ,       __lowerCamelCase    )

									with tempfile.TemporaryDirectory() as tmp_dir:
											shutil.copy('''./tests/fixtures/vocab.json'''       ,       os.path.join(__lowerCamelCase       ,       '''vocab.json'''    )    )
											shutil.copy('''./tests/fixtures/merges.txt'''       ,       os.path.join(__lowerCamelCase       ,       '''merges.txt'''    )    )

											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase       ,       tokenizer_type='''gpt2'''       ,       use_fast=__lowerCamelCase    )
											self.assertIsInstance(__lowerCamelCase       ,       __lowerCamelCase    )

							@require_tokenizers
							def _lowercase					(self    ):

									"""simple docstring"""







									with tempfile.TemporaryDirectory() as tmp_dir:
											shutil.copy('''./tests/fixtures/vocab.txt'''       ,       os.path.join(__lowerCamelCase       ,       '''vocab.txt'''    )    )

											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase       ,       tokenizer_type='''bert'''    )
											self.assertIsInstance(__lowerCamelCase       ,       __lowerCamelCase    )

									with tempfile.TemporaryDirectory() as tmp_dir:
											shutil.copy('''./tests/fixtures/vocab.json'''       ,       os.path.join(__lowerCamelCase       ,       '''vocab.json'''    )    )
											shutil.copy('''./tests/fixtures/merges.txt'''       ,       os.path.join(__lowerCamelCase       ,       '''merges.txt'''    )    )

											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase       ,       tokenizer_type='''gpt2'''    )
											self.assertIsInstance(__lowerCamelCase       ,       __lowerCamelCase    )

							def _lowercase					(self    ):

									"""simple docstring"""







									with pytest.raises(__lowerCamelCase    ):
											AutoTokenizer.from_pretrained('''./'''       ,       tokenizer_type='''xxx'''    )

							@require_tokenizers
							def _lowercase					(self    ):

									"""simple docstring"""







									for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
											SCREAMING_SNAKE_CASE_						 = tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased'''    )
											self.assertIsInstance(__lowerCamelCase       ,       (BertTokenizer, BertTokenizerFast)    )

											if isinstance(__lowerCamelCase       ,       __lowerCamelCase    ):
													self.assertEqual(tokenizer.basic_tokenizer.do_lower_case       ,       __lowerCamelCase    )
											else:
													self.assertEqual(tokenizer.do_lower_case       ,       __lowerCamelCase    )

											self.assertEqual(tokenizer.model_max_length       ,       5_12    )

							@require_tokenizers
							def _lowercase					(self    ):

									"""simple docstring"""







									for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
											with self.assertRaisesRegex(
											    __lowerCamelCase       ,       '''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''       ,       ):
													SCREAMING_SNAKE_CASE_						 = tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists'''    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TOKENIZER_MAPPING.values()
									SCREAMING_SNAKE_CASE_						 = []

									for slow_tok, fast_tok in tokenizers:
											if slow_tok is not None:
													tokenizer_names.append(slow_tok.__name__    )

											if fast_tok is not None:
													tokenizer_names.append(fast_tok.__name__    )

									for tokenizer_name in tokenizer_names:
											# must find the right class
											tokenizer_class_from_name(__lowerCamelCase    )

							@require_tokenizers
							def _lowercase					(self    ):

									"""simple docstring"""







									self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''       ,       use_fast=__lowerCamelCase    )       ,       __lowerCamelCase    )
									self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''    )       ,       __lowerCamelCase    )

							@require_tokenizers
							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained('''distilbert-base-uncased'''       ,       do_lower_case=__lowerCamelCase    )
									SCREAMING_SNAKE_CASE_						 = "Hello, world. How are you?"
									SCREAMING_SNAKE_CASE_						 = tokenizer.tokenize(__lowerCamelCase    )
									self.assertEqual('''[UNK]'''       ,       tokens[0]    )

									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''       ,       do_lower_case=__lowerCamelCase    )
									SCREAMING_SNAKE_CASE_						 = tokenizer.tokenize(__lowerCamelCase    )
									self.assertEqual('''[UNK]'''       ,       tokens[0]    )

							@require_tokenizers
							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config'''    )
									self.assertEqual(type(__lowerCamelCase    )       ,       __lowerCamelCase    )
									self.assertEqual(tokenizer.model_max_length       ,       5_12    )
									self.assertEqual(tokenizer.vocab_size       ,       3_00_00    )
									self.assertEqual(tokenizer.unk_token       ,       '''[UNK]'''    )
									self.assertEqual(tokenizer.padding_side       ,       '''right'''    )
									self.assertEqual(tokenizer.truncation_side       ,       '''right'''    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase    )
									self.assertIsInstance(__lowerCamelCase       ,       (BertTokenizer, BertTokenizerFast)    )
									with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(__lowerCamelCase    )
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase    )

									self.assertIsInstance(__lowerCamelCase       ,       tokenizer.__class__    )
									self.assertEqual(tokenizera.vocab_size       ,       12    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained('''ctrl'''    )
									# There is no fast CTRL so this always gives us a slow tokenizer.
									self.assertIsInstance(__lowerCamelCase       ,       __lowerCamelCase    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = get_tokenizer_config('''bert-base-cased'''    )
									SCREAMING_SNAKE_CASE_						 = config.pop('''_commit_hash'''       ,       __lowerCamelCase    )
									# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
									self.assertEqual(__lowerCamelCase       ,       {'''do_lower_case''': False}    )

									# This model does not have a tokenizer_config so we get back an empty dict.
									SCREAMING_SNAKE_CASE_						 = get_tokenizer_config(__lowerCamelCase    )
									self.assertDictEqual(__lowerCamelCase       ,       {}    )

									# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase    )
									with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(__lowerCamelCase    )
											SCREAMING_SNAKE_CASE_						 = get_tokenizer_config(__lowerCamelCase    )

									# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
									self.assertEqual(config['''tokenizer_class''']       ,       '''BertTokenizer'''    )

							def _lowercase					(self    ):

									"""simple docstring"""







									try:
											AutoConfig.register('''custom'''       ,       __lowerCamelCase    )

											AutoTokenizer.register(__lowerCamelCase       ,       slow_tokenizer_class=__lowerCamelCase    )
											# Trying to register something existing in the Transformers library will raise an error
											with self.assertRaises(__lowerCamelCase    ):
													AutoTokenizer.register(__lowerCamelCase       ,       slow_tokenizer_class=__lowerCamelCase    )

											SCREAMING_SNAKE_CASE_						 = CustomTokenizer.from_pretrained(__lowerCamelCase    )
											with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(__lowerCamelCase    )

													SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase    )
													self.assertIsInstance(__lowerCamelCase       ,       __lowerCamelCase    )

									finally:
											if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
											if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]

							@require_tokenizers
							def _lowercase					(self    ):

									"""simple docstring"""







									try:
											AutoConfig.register('''custom'''       ,       __lowerCamelCase    )

											# Can register in two steps
											AutoTokenizer.register(__lowerCamelCase       ,       slow_tokenizer_class=__lowerCamelCase    )
											self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,       (CustomTokenizer, None)    )
											AutoTokenizer.register(__lowerCamelCase       ,       fast_tokenizer_class=__lowerCamelCase    )
											self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,       (CustomTokenizer, CustomTokenizerFast)    )

											del TOKENIZER_MAPPING._extra_content[CustomConfig]
											# Can register in one step
											AutoTokenizer.register(
											    __lowerCamelCase       ,       slow_tokenizer_class=__lowerCamelCase       ,       fast_tokenizer_class=__lowerCamelCase    )
											self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,       (CustomTokenizer, CustomTokenizerFast)    )

											# Trying to register something existing in the Transformers library will raise an error
											with self.assertRaises(__lowerCamelCase    ):
													AutoTokenizer.register(__lowerCamelCase       ,       fast_tokenizer_class=__lowerCamelCase    )

											# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
											# and that model does not have a tokenizer.json
											with tempfile.TemporaryDirectory() as tmp_dir:
													SCREAMING_SNAKE_CASE_						 = BertTokenizerFast.from_pretrained(__lowerCamelCase    )
													bert_tokenizer.save_pretrained(__lowerCamelCase    )
													SCREAMING_SNAKE_CASE_						 = CustomTokenizerFast.from_pretrained(__lowerCamelCase    )

											with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(__lowerCamelCase    )

													SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase    )
													self.assertIsInstance(__lowerCamelCase       ,       __lowerCamelCase    )

													SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase       ,       use_fast=__lowerCamelCase    )
													self.assertIsInstance(__lowerCamelCase       ,       __lowerCamelCase    )

									finally:
											if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
											if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]

							def _lowercase					(self    ):

									"""simple docstring"""







									with self.assertRaises(__lowerCamelCase    ):
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''    )
									# If remote code is disabled, we can't load this config.
									with self.assertRaises(__lowerCamelCase    ):
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer'''       ,       trust_remote_code=__lowerCamelCase    )

									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''       ,       trust_remote_code=__lowerCamelCase    )
									self.assertTrue(tokenizer.special_attribute_present    )
									# Test tokenizer can be reloaded.
									with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(__lowerCamelCase    )
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase       ,       trust_remote_code=__lowerCamelCase    )
									self.assertTrue(reloaded_tokenizer.special_attribute_present    )

									if is_tokenizers_available():
											self.assertEqual(tokenizer.__class__.__name__       ,       '''NewTokenizerFast'''    )
											self.assertEqual(reloaded_tokenizer.__class__.__name__       ,       '''NewTokenizerFast'''    )

											# Test we can also load the slow version
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer'''       ,       trust_remote_code=__lowerCamelCase       ,       use_fast=__lowerCamelCase    )
											self.assertTrue(tokenizer.special_attribute_present    )
											self.assertEqual(tokenizer.__class__.__name__       ,       '''NewTokenizer'''    )
											# Test tokenizer can be reloaded.
											with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(__lowerCamelCase    )
													SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase       ,       trust_remote_code=__lowerCamelCase       ,       use_fast=__lowerCamelCase    )
											self.assertEqual(reloaded_tokenizer.__class__.__name__       ,       '''NewTokenizer'''    )
											self.assertTrue(reloaded_tokenizer.special_attribute_present    )
									else:
											self.assertEqual(tokenizer.__class__.__name__       ,       '''NewTokenizer'''    )
											self.assertEqual(reloaded_tokenizer.__class__.__name__       ,       '''NewTokenizer'''    )

							@require_tokenizers
							def _lowercase					(self    ):

									"""simple docstring"""







									class  snake_case     (							_A   ):
																UpperCAmelCase__						     =      False







									class  snake_case     (							_A   ):
																UpperCAmelCase__						     =      NewTokenizer
																UpperCAmelCase__						     =      False

									try:
											AutoConfig.register('''custom'''       ,       __lowerCamelCase    )
											AutoTokenizer.register(__lowerCamelCase       ,       slow_tokenizer_class=__lowerCamelCase    )
											AutoTokenizer.register(__lowerCamelCase       ,       fast_tokenizer_class=__lowerCamelCase    )
											# If remote code is not set, the default is to use local
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''    )
											self.assertEqual(tokenizer.__class__.__name__       ,       '''NewTokenizerFast'''    )
											self.assertFalse(tokenizer.special_attribute_present    )
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''       ,       use_fast=__lowerCamelCase    )
											self.assertEqual(tokenizer.__class__.__name__       ,       '''NewTokenizer'''    )
											self.assertFalse(tokenizer.special_attribute_present    )

											# If remote code is disabled, we load the local one.
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer'''       ,       trust_remote_code=__lowerCamelCase    )
											self.assertEqual(tokenizer.__class__.__name__       ,       '''NewTokenizerFast'''    )
											self.assertFalse(tokenizer.special_attribute_present    )
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer'''       ,       trust_remote_code=__lowerCamelCase       ,       use_fast=__lowerCamelCase    )
											self.assertEqual(tokenizer.__class__.__name__       ,       '''NewTokenizer'''    )
											self.assertFalse(tokenizer.special_attribute_present    )

											# If remote is enabled, we load from the Hub
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer'''       ,       trust_remote_code=__lowerCamelCase    )
											self.assertEqual(tokenizer.__class__.__name__       ,       '''NewTokenizerFast'''    )
											self.assertTrue(tokenizer.special_attribute_present    )
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer'''       ,       trust_remote_code=__lowerCamelCase       ,       use_fast=__lowerCamelCase    )
											self.assertEqual(tokenizer.__class__.__name__       ,       '''NewTokenizer'''    )
											self.assertTrue(tokenizer.special_attribute_present    )

									finally:
											if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
											if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(
									    '''hf-internal-testing/test_dynamic_tokenizer_legacy'''       ,       trust_remote_code=__lowerCamelCase    )
									self.assertTrue(tokenizer.special_attribute_present    )
									if is_tokenizers_available():
											self.assertEqual(tokenizer.__class__.__name__       ,       '''NewTokenizerFast'''    )

											# Test we can also load the slow version
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(
											    '''hf-internal-testing/test_dynamic_tokenizer_legacy'''       ,       trust_remote_code=__lowerCamelCase       ,       use_fast=__lowerCamelCase    )
											self.assertTrue(tokenizer.special_attribute_present    )
											self.assertEqual(tokenizer.__class__.__name__       ,       '''NewTokenizer'''    )
									else:
											self.assertEqual(tokenizer.__class__.__name__       ,       '''NewTokenizer'''    )

							def _lowercase					(self    ):

									"""simple docstring"""







									with self.assertRaisesRegex(
									    __lowerCamelCase       ,       '''bert-base is not a local folder and is not a valid model identifier'''    ):
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained('''bert-base'''    )

							def _lowercase					(self    ):

									"""simple docstring"""







									with self.assertRaisesRegex(
									    __lowerCamelCase       ,       R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''    ):
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(__lowerCamelCase       ,       revision='''aaaaaa'''    )







							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''    )
									with RequestCounter() as counter:
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''    )
											self.assertEqual(counter.get_request_count       ,       0    )
											self.assertEqual(counter.head_request_count       ,       1    )
											self.assertEqual(counter.other_request_count       ,       0    )

"""simple docstring"""




import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
						from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
						import torch_xla.core.xla_model as xm



def        _lowerCamelCase ( __a    ):
		if is_torch_version('''<''',							'''2.0.0'''    ) or not hasattr(__a,							'''_dynamo'''    ):
				return False
		return isinstance(__a,							torch._dynamo.eval_frame.OptimizedModule    )



def        _lowerCamelCase ( __a,							__a = True    ):
		SCREAMING_SNAKE_CASE_						 = (torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

		SCREAMING_SNAKE_CASE_						 = is_compiled_module(__a    )
		if is_compiled:
				SCREAMING_SNAKE_CASE_						 = model
				SCREAMING_SNAKE_CASE_						 = model._orig_mod

		if is_deepspeed_available():
				options += (DeepSpeedEngine,)

		while isinstance(__a,							__a    ):
				SCREAMING_SNAKE_CASE_						 = model.module

		if not keep_fpaa_wrapper:
				SCREAMING_SNAKE_CASE_						 = getattr(__a,							'''forward'''    )
				SCREAMING_SNAKE_CASE_						 = model.__dict__.pop('''_original_forward''',							__a    )
				if original_forward is not None:
						while hasattr(__a,							'''__wrapped__'''    ):
								SCREAMING_SNAKE_CASE_						 = forward.__wrapped__
								if forward == original_forward:
										break
						SCREAMING_SNAKE_CASE_						 = forward
				if getattr(__a,							'''_converted_to_transformer_engine''',							__a    ):
						convert_model(__a,							to_transformer_engine=__a    )

		if is_compiled:
				SCREAMING_SNAKE_CASE_						 = model
				SCREAMING_SNAKE_CASE_						 = compiled_model

		return model



def        _lowerCamelCase ( ):
		PartialState().wait_for_everyone()



def        _lowerCamelCase ( __a,							__a    ):
		if PartialState().distributed_type == DistributedType.TPU:
				xm.save(__a,							__a    )
		elif PartialState().local_process_index == 0:
				torch.save(__a,							__a    )



@contextmanager
def        _lowerCamelCase ( **__a    ):
		for key, value in kwargs.items():
				SCREAMING_SNAKE_CASE_						 = str(__a    )

		yield

		for key in kwargs:
				if key.upper() in os.environ:
						del os.environ[key.upper()]



def        _lowerCamelCase ( __a    ):
		if not hasattr(__a,							'''__qualname__'''    ) and not hasattr(__a,							'''__name__'''    ):
				SCREAMING_SNAKE_CASE_						 = getattr(__a,							'''__class__''',							__a    )
		if hasattr(__a,							'''__qualname__'''    ):
				return obj.__qualname__
		if hasattr(__a,							'''__name__'''    ):
				return obj.__name__
		return str(__a    )



def        _lowerCamelCase ( __a,							__a    ):
		for key, value in source.items():
				if isinstance(__a,							__a    ):
						SCREAMING_SNAKE_CASE_						 = destination.setdefault(__a,							{}    )
						merge_dicts(__a,							__a    )
				else:
						SCREAMING_SNAKE_CASE_						 = value

		return destination



def        _lowerCamelCase ( __a = None    ):
		if port is None:
				SCREAMING_SNAKE_CASE_						 = 29_500
		with socket.socket(socket.AF_INET,							socket.SOCK_STREAM    ) as s:
				return s.connect_ex(('''localhost''', port)    ) == 0