

import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


lowerCAmelCase__	   = [
    {"""dataset""": """wikipedia""", """config_name""": """20220301.de"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.en"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.fr"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.frr"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.it"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.simple"""},
    {"""dataset""": """snli""", """config_name""": """plain_text"""},
    {"""dataset""": """eli5""", """config_name""": """LFQA_reddit"""},
    {"""dataset""": """wiki40b""", """config_name""": """en"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.nq.compressed"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.nq.no_index"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.multiset.no_index"""},
    {"""dataset""": """natural_questions""", """config_name""": """default"""},
]



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							str=True      )			->      Union[str, Any]:




				'''simple docstring'''





				if with_config:
								return [
								    {
								        "testcase_name": d["dataset"] + "/" + d["config_name"],
								        "dataset": d["dataset"],
								        "config_name": d["config_name"],
								    }
								    for d in DATASETS_ON_HF_GCP
								]
				else:
								return [
								    {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
								]

@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=snake_case			)			)
class 					a__						(					snake_case			):
			"""simple docstring"""







			__lowerCamelCase        =       None
			__lowerCamelCase        =       None


			def      UpperCamelCase		(    self      ,      lowercase      ,      lowercase       )  ->						Dict:
							'''simple docstring'''






							with TemporaryDirectory() as tmp_dir:
											A__        =			dataset_module_factory(lowercase      ,      cache_dir=lowercase       )

											A__        =			import_main_class(dataset_module.module_path      ,      dataset=lowercase       )

											A__        =			builder_cls(
											    cache_dir=lowercase      ,      config_name=lowercase      ,      hash=dataset_module.hash      ,      )

											A__        =			"/".join(
											    [
											        HF_GCP_BASE_URL,
											        builder_instance._relative_data_dir(with_hash=lowercase       ).replace(os.sep      ,      "/"       ),
											        config.DATASET_INFO_FILENAME,
											    ]       )
											A__        =			cached_path(lowercase      ,      cache_dir=lowercase       )
											self.assertTrue(os.path.exists(lowercase       )       )



@pytest.mark.integration
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Optional[int]      )			->      Any:




				'''simple docstring'''





				A__        =			tmp_path_factory.mktemp("test_hf_gcp"      ) / "test_wikipedia_simple"
				A__        =			dataset_module_factory("wikipedia"	,       cache_dir=SCREAMING_SNAKE_CASE_      )
				A__        =			import_main_class(dataset_module.module_path      )
				A__        =			builder_cls(
				    cache_dir=SCREAMING_SNAKE_CASE_	,       config_name="20220301.frr"	,       hash=dataset_module.hash	,       )
				# use the HF cloud storage, not the original download_and_prepare that uses apache-beam
				A__        =			None
				builder_instance.download_and_prepare()
				A__        =			builder_instance.as_dataset()
				assert ds





@pytest.mark.integration
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							List[Any]      )			->      Optional[Any]:




				'''simple docstring'''





				A__        =			dataset_module_factory("wikipedia"	,       cache_dir=SCREAMING_SNAKE_CASE_      )
				A__        =			import_main_class(dataset_module.module_path	,       dataset=SCREAMING_SNAKE_CASE_      )
				A__        =			builder_cls(
				    cache_dir=SCREAMING_SNAKE_CASE_	,       config_name="20220301.frr"	,       hash=dataset_module.hash	,       )
				A__        =			builder_instance.as_streaming_dataset()
				assert ds
				assert isinstance(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      )
				assert "train" in ds
				assert isinstance(ds["train"]	,       SCREAMING_SNAKE_CASE_      )
				assert next(iter(ds["train"]      )      )




import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
lowerCAmelCase__	   = """src/transformers"""


# This is to make sure the transformers module imported is the one in the repo.
lowerCAmelCase__	   = direct_transformers_import(PATH_TO_TRANSFORMERS)

lowerCAmelCase__	   = transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
lowerCAmelCase__	   = re.compile(R"""\[(.+?)\]\((https://huggingface\.co/.+?)\)""")


lowerCAmelCase__	   = {
    """DecisionTransformerConfig""",
    """EncoderDecoderConfig""",
    """MusicgenConfig""",
    """RagConfig""",
    """SpeechEncoderDecoderConfig""",
    """TimmBackboneConfig""",
    """VisionEncoderDecoderConfig""",
    """VisionTextDualEncoderConfig""",
    """LlamaConfig""",
}



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Union[str, Any]      )			->      Dict:




				'''simple docstring'''





				A__        =			None

				# source code of `config_class`
				A__        =			inspect.getsource(SCREAMING_SNAKE_CASE_      )
				A__        =			_re_checkpoint.findall(SCREAMING_SNAKE_CASE_      )

				# Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
				# For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
				for ckpt_name, ckpt_link in checkpoints:
								# allow the link to end with `/`
								if ckpt_link.endswith("/"      ):
												A__        =			ckpt_link[:-1]

								# verify the checkpoint name corresponds to the checkpoint link
								A__        =			F'https://huggingface.co/{ckpt_name}'
								if ckpt_link == ckpt_link_from_name:
												A__        =			ckpt_name
												break

				return checkpoint



def 					lowerCAmelCase__							(	)			->      List[str]:




				'''simple docstring'''





				A__        =			[]

				for config_class in list(CONFIG_MAPPING.values()      ):
								# Skip deprecated models
								if "models.deprecated" in config_class.__module__:
												continue
								A__        =			get_checkpoint_from_config_class(SCREAMING_SNAKE_CASE_      )

								A__        =			config_class.__name__
								if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
												configs_without_checkpoint.append(SCREAMING_SNAKE_CASE_      )

				if len(SCREAMING_SNAKE_CASE_      ) > 0:
								A__        =			"\n".join(sorted(SCREAMING_SNAKE_CASE_      )      )
								raise ValueError(F'The following configurations don\'t contain any valid checkpoint:\n{message}'      )


if __name__ == "__main__":
	check_config_docstrings_have_checkpoints()

