

import os
from typing import Optional

import fsspec
from fsspec.archive import AbstractArchiveFileSystem
from fsspec.utils import DEFAULT_BLOCK_SIZE

class 					a__						(					snake_case			):
			"""simple docstring"""







			__lowerCamelCase        =       ''
			__lowerCamelCase        =       (
			    None  # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
			)
			__lowerCamelCase        =       None  # compression type in fsspec. ex: "gzip"
			__lowerCamelCase        =       None  # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz


			def __init__(    self      ,      lowercase = ""      ,      lowercase = None      ,      lowercase = None      ,      **lowercase       )  ->						Any:
							'''simple docstring'''






							super().__init__(self      ,      **lowercase       )
							# always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
							A__        =			fsspec.open(
							    lowercase      ,      mode="rb"      ,      protocol=lowercase      ,      compression=self.compression      ,      client_kwargs={
							        "requote_redirect_url": False,  # see https://github.com/huggingface/datasets/pull/5459
							        "trust_env": True,  # Enable reading proxy env variables.
							        **(target_options or {}).pop("client_kwargs"      ,      {}       ),  # To avoid issues if it was already passed.
							    }      ,      **(target_options or {})      ,      )
							A__        =			os.path.basename(self.file.path.split("::"       )[0]       )
							A__        =			(
							    self.compressed_name[: self.compressed_name.rindex("."       )]
							    if "." in self.compressed_name
							    else self.compressed_name
							)
							A__        =			None


			@classmethod
			def      UpperCamelCase		(    cls      ,      lowercase       )  ->						Tuple:
							'''simple docstring'''






							return super()._strip_protocol(lowercase       ).lstrip("/"       )


			def      UpperCamelCase		(    self       )  ->						int:
							'''simple docstring'''






							if self.dir_cache is None:
											A__        =			{**self.file.fs.info(self.file.path       ), "name": self.uncompressed_name}
											A__        =			{f["name"]: f}


			def      UpperCamelCase		(    self      ,      lowercase       )  ->						Tuple:
							'''simple docstring'''






							return self.file.open().read()


			def      UpperCamelCase		(    self      ,      lowercase      ,      lowercase = "rb"      ,      lowercase=None      ,      lowercase=True      ,      lowercase=None      ,      **lowercase      ,      )  ->						Union[str, Any]:
							'''simple docstring'''






							A__        =			self._strip_protocol(lowercase       )
							if mode != "rb":
											raise ValueError(F'Tried to read with mode {mode} on file {self.file.path} opened with mode \'rb\''       )
							return self.file.open()

class 					a__						(					snake_case			):
			"""simple docstring"""







			__lowerCamelCase        =       'bz2'
			__lowerCamelCase        =       'bz2'
			__lowerCamelCase        =       '.bz2'

class 					a__						(					snake_case			):
			"""simple docstring"""







			__lowerCamelCase        =       'gzip'
			__lowerCamelCase        =       'gzip'
			__lowerCamelCase        =       '.gz'

class 					a__						(					snake_case			):
			"""simple docstring"""







			__lowerCamelCase        =       'lz4'
			__lowerCamelCase        =       'lz4'
			__lowerCamelCase        =       '.lz4'

class 					a__						(					snake_case			):
			"""simple docstring"""







			__lowerCamelCase        =       'xz'
			__lowerCamelCase        =       'xz'
			__lowerCamelCase        =       '.xz'

class 					a__						(					snake_case			):
			"""simple docstring"""







			__lowerCamelCase        =       'zstd'
			__lowerCamelCase        =       'zstd'
			__lowerCamelCase        =       '.zst'


			def __init__(    self      ,      lowercase      ,      lowercase = "rb"      ,      lowercase = None      ,      lowercase = None      ,      lowercase = DEFAULT_BLOCK_SIZE      ,      **lowercase      ,      )  ->						List[str]:
							'''simple docstring'''






							super().__init__(
							    fo=lowercase      ,      mode=lowercase      ,      target_protocol=lowercase      ,      target_options=lowercase      ,      block_size=lowercase      ,      **lowercase      ,      )
							# We need to wrap the zstd decompressor to avoid this error in fsspec==2021.7.0 and zstandard==0.15.2:
							#
							# File "/Users/user/.virtualenvs/hf-datasets/lib/python3.7/site-packages/fsspec/core.py", line 145, in open
							#     out.close = close
							# AttributeError: 'zstd.ZstdDecompressionReader' object attribute 'close' is read-only
							#
							# see https://github.com/intake/filesystem_spec/issues/725
							A__        =			self.file.__enter__

							class 					a__						:
										"""simple docstring"""







										def __init__(    self      ,      lowercase       )  ->						List[Any]:
														'''simple docstring'''






														A__        =			file_


										def __enter__(    self       )  ->						List[str]:
														'''simple docstring'''






														self._file.__enter__()
														return self


										def __exit__(    self      ,      *lowercase      ,      **lowercase       )  ->						Union[str, Any]:
														'''simple docstring'''






														self._file.__exit__(*lowercase      ,      **lowercase       )


										def __iter__(    self       )  ->						Union[str, Any]:
														'''simple docstring'''






														return iter(self._file       )


										def      UpperCamelCase		(    self       )  ->						List[str]:
														'''simple docstring'''






														return next(self._file       )


										def __getattr__(    self      ,      lowercase       )  ->						Union[str, Any]:
														'''simple docstring'''






														return getattr(self._file      ,      lowercase       )

							def fixed_enter(*lowercase      ,      **lowercase       ):
											return WrappedFile(_enter(*lowercase      ,      **lowercase       )       )

							A__        =			fixed_enter




from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
	from transformers import (
	    TFAutoModel,
	    TFAutoModelForCausalLM,
	    TFAutoModelForMaskedLM,
	    TFAutoModelForPreTraining,
	    TFAutoModelForQuestionAnswering,
	    TFAutoModelForSeqaSeqLM,
	    TFAutoModelForSequenceClassification,
	    TFAutoModelForTableQuestionAnswering,
	    TFAutoModelForTokenClassification,
	    TFAutoModelWithLMHead,
	    TFBertForMaskedLM,
	    TFBertForPreTraining,
	    TFBertForQuestionAnswering,
	    TFBertForSequenceClassification,
	    TFBertModel,
	    TFFunnelBaseModel,
	    TFFunnelModel,
	    TFGPTaLMHeadModel,
	    TFRobertaForMaskedLM,
	    TFTaForConditionalGeneration,
	    TFTapasForQuestionAnswering,
	)
	from transformers.models.auto.modeling_tf_auto import (
	    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
	    TF_MODEL_FOR_MASKED_LM_MAPPING,
	    TF_MODEL_FOR_PRETRAINING_MAPPING,
	    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
	    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
	    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
	    TF_MODEL_MAPPING,
	)
	from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
	from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
	from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
	from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST

class 					a__						(					snake_case			):
			"""simple docstring"""







			__lowerCamelCase        =       'new-model'


if is_tf_available():

	class 					a__						(					snake_case			):
				"""simple docstring"""







				__lowerCamelCase        =       NewModelConfig

@require_tf
class 					a__						(					unittest.TestCase			):
			"""simple docstring"""







			@slow
			def      UpperCamelCase		(    self       )  ->						List[Any]:
							'''simple docstring'''






							A__        =			"bert-base-cased"
							A__        =			AutoConfig.from_pretrained(lowercase       )
							self.assertIsNotNone(lowercase       )
							self.assertIsInstance(lowercase      ,      lowercase       )

							A__        =			TFAutoModel.from_pretrained(lowercase       )
							self.assertIsNotNone(lowercase       )
							self.assertIsInstance(lowercase      ,      lowercase       )


			@slow
			def      UpperCamelCase		(    self       )  ->						Optional[Any]:
							'''simple docstring'''






							A__        =			"bert-base-cased"
							A__        =			AutoConfig.from_pretrained(lowercase       )
							self.assertIsNotNone(lowercase       )
							self.assertIsInstance(lowercase      ,      lowercase       )

							A__        =			TFAutoModelForPreTraining.from_pretrained(lowercase       )
							self.assertIsNotNone(lowercase       )
							self.assertIsInstance(lowercase      ,      lowercase       )


			@slow
			def      UpperCamelCase		(    self       )  ->						Dict:
							'''simple docstring'''






							for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											A__        =			AutoConfig.from_pretrained(lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )

											A__        =			TFAutoModelForCausalLM.from_pretrained(lowercase       )
											A__    ,				A__        =			TFAutoModelForCausalLM.from_pretrained(lowercase      ,      output_loading_info=lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )


			@slow
			def      UpperCamelCase		(    self       )  ->						Optional[int]:
							'''simple docstring'''






							for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											A__        =			AutoConfig.from_pretrained(lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )

											A__        =			TFAutoModelWithLMHead.from_pretrained(lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )


			@slow
			def      UpperCamelCase		(    self       )  ->						int:
							'''simple docstring'''






							for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											A__        =			AutoConfig.from_pretrained(lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )

											A__        =			TFAutoModelForMaskedLM.from_pretrained(lowercase       )
											A__    ,				A__        =			TFAutoModelForMaskedLM.from_pretrained(lowercase      ,      output_loading_info=lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )


			@slow
			def      UpperCamelCase		(    self       )  ->						str:
							'''simple docstring'''






							for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											A__        =			AutoConfig.from_pretrained(lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )

											A__        =			TFAutoModelForSeqaSeqLM.from_pretrained(lowercase       )
											A__    ,				A__        =			TFAutoModelForSeqaSeqLM.from_pretrained(lowercase      ,      output_loading_info=lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )


			@slow
			def      UpperCamelCase		(    self       )  ->						Dict:
							'''simple docstring'''






							for model_name in ["bert-base-uncased"]:
											A__        =			AutoConfig.from_pretrained(lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )

											A__        =			TFAutoModelForSequenceClassification.from_pretrained(lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )


			@slow
			def      UpperCamelCase		(    self       )  ->						int:
							'''simple docstring'''






							for model_name in ["bert-base-uncased"]:
											A__        =			AutoConfig.from_pretrained(lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )

											A__        =			TFAutoModelForQuestionAnswering.from_pretrained(lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )


			@slow
			@require_tensorflow_probability
			def      UpperCamelCase		(    self       )  ->						List[str]:
							'''simple docstring'''






							for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
											A__        =			AutoConfig.from_pretrained(lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )

											A__        =			TFAutoModelForTableQuestionAnswering.from_pretrained(lowercase       )
											A__    ,				A__        =			TFAutoModelForTableQuestionAnswering.from_pretrained(
											    lowercase      ,      output_loading_info=lowercase       )
											self.assertIsNotNone(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )


			def      UpperCamelCase		(    self       )  ->						Dict:
							'''simple docstring'''






							A__        =			TFAutoModelWithLMHead.from_pretrained(lowercase       )
							self.assertIsInstance(lowercase      ,      lowercase       )
							self.assertEqual(model.num_parameters()      ,      14410       )
							self.assertEqual(model.num_parameters(only_trainable=lowercase       )      ,      14410       )


			def      UpperCamelCase		(    self       )  ->						Any:
							'''simple docstring'''






							A__        =			TFAutoModelWithLMHead.from_pretrained(lowercase       )
							self.assertIsInstance(lowercase      ,      lowercase       )
							self.assertEqual(model.num_parameters()      ,      14410       )
							self.assertEqual(model.num_parameters(only_trainable=lowercase       )      ,      14410       )


			def      UpperCamelCase		(    self       )  ->						int:
							'''simple docstring'''






							A__        =			TFAutoModel.from_pretrained("sgugger/funnel-random-tiny"       )
							self.assertIsInstance(lowercase      ,      lowercase       )

							A__        =			copy.deepcopy(model.config       )
							A__        =			["FunnelBaseModel"]
							A__        =			TFAutoModel.from_config(lowercase       )
							self.assertIsInstance(lowercase      ,      lowercase       )

							with tempfile.TemporaryDirectory() as tmp_dir:
											model.save_pretrained(lowercase       )
											A__        =			TFAutoModel.from_pretrained(lowercase       )
											self.assertIsInstance(lowercase      ,      lowercase       )


			def      UpperCamelCase		(    self       )  ->						Optional[Any]:
							'''simple docstring'''






							try:
											AutoConfig.register("new-model"      ,      lowercase       )

											A__        =			[
											    TFAutoModel,
											    TFAutoModelForCausalLM,
											    TFAutoModelForMaskedLM,
											    TFAutoModelForPreTraining,
											    TFAutoModelForQuestionAnswering,
											    TFAutoModelForSequenceClassification,
											    TFAutoModelForTokenClassification,
											]

											for auto_class in auto_classes:
															with self.subTest(auto_class.__name__       ):
																			# Wrong config class will raise an error
																			with self.assertRaises(lowercase       ):
																							auto_class.register(lowercase      ,      lowercase       )
																			auto_class.register(lowercase      ,      lowercase       )
																			# Trying to register something existing in the Transformers library will raise an error
																			with self.assertRaises(lowercase       ):
																							auto_class.register(lowercase      ,      lowercase       )

																			# Now that the config is registered, it can be used as any other config with the auto-API
																			A__        =			BertModelTester(self       ).get_config()
																			A__        =			NewModelConfig(**tiny_config.to_dict()       )
																			A__        =			auto_class.from_config(lowercase       )
																			self.assertIsInstance(lowercase      ,      lowercase       )

																			with tempfile.TemporaryDirectory() as tmp_dir:
																							model.save_pretrained(lowercase       )
																							A__        =			auto_class.from_pretrained(lowercase       )
																							self.assertIsInstance(lowercase      ,      lowercase       )

							finally:
											if "new-model" in CONFIG_MAPPING._extra_content:
															del CONFIG_MAPPING._extra_content["new-model"]
											for mapping in (
											    TF_MODEL_MAPPING,
											    TF_MODEL_FOR_PRETRAINING_MAPPING,
											    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
											    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
											    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
											    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
											    TF_MODEL_FOR_MASKED_LM_MAPPING,
											):
															if NewModelConfig in mapping._extra_content:
																			del mapping._extra_content[NewModelConfig]


			def      UpperCamelCase		(    self       )  ->						str:
							'''simple docstring'''






							with self.assertRaisesRegex(
							    lowercase      ,      "bert-base is not a local folder and is not a valid model identifier"       ):
											A__        =			TFAutoModel.from_pretrained("bert-base"       )


			def      UpperCamelCase		(    self       )  ->						Optional[Any]:
							'''simple docstring'''






							with self.assertRaisesRegex(
							    lowercase      ,      R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"       ):
											A__        =			TFAutoModel.from_pretrained(lowercase      ,      revision="aaaaaa"       )


			def      UpperCamelCase		(    self       )  ->						Union[str, Any]:
							'''simple docstring'''






							with self.assertRaisesRegex(
							    lowercase      ,      "hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin"      ,      ):
											A__        =			TFAutoModel.from_pretrained("hf-internal-testing/config-no-model"       )


			def      UpperCamelCase		(    self       )  ->						Union[str, Any]:
							'''simple docstring'''






							with self.assertRaisesRegex(lowercase      ,      "Use `from_pt=True` to load this model"       ):
											A__        =			TFAutoModel.from_pretrained("hf-internal-testing/tiny-bert-pt-only"       )


			def      UpperCamelCase		(    self       )  ->						Any:
							'''simple docstring'''






							A__        =			TFAutoModel.from_pretrained("hf-internal-testing/tiny-random-bert"       )
							with RequestCounter() as counter:
											A__        =			TFAutoModel.from_pretrained("hf-internal-testing/tiny-random-bert"       )
											self.assertEqual(counter.get_request_count      ,      0       )
											self.assertEqual(counter.head_request_count      ,      1       )
											self.assertEqual(counter.other_request_count      ,      0       )

							# With a sharded checkpoint
							A__        =			TFAutoModel.from_pretrained("ArthurZ/tiny-random-bert-sharded"       )
							with RequestCounter() as counter:
											A__        =			TFAutoModel.from_pretrained("ArthurZ/tiny-random-bert-sharded"       )
											self.assertEqual(counter.get_request_count      ,      0       )
											self.assertEqual(counter.head_request_count      ,      1       )
											self.assertEqual(counter.other_request_count      ,      0       )

