

import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


lowerCAmelCase__	   = """http://www.mocksite.com/file1.txt"""
lowerCAmelCase__	   = """\"text\": [\"foo\", \"foo\"]"""
lowerCAmelCase__	   = """6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8"""

class 					a__						:
			"""simple docstring"""







			__lowerCamelCase        =       200
			__lowerCamelCase        =       {'Content-Length': '100'}
			__lowerCamelCase        =       {}


			def      UpperCamelCase		(    self      ,      **lowercase       )  ->						int:
							'''simple docstring'''






							return [bytes(lowercase      ,      "utf-8"       )]



def 					lowerCAmelCase__							(	*SCREAMING_SNAKE_CASE_:							Any	,       **SCREAMING_SNAKE_CASE_:							Any      )			->      int:




				'''simple docstring'''





				return MockResponse()



@pytest.mark.parametrize("urls_type"	,       [str, list, dict]      )
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Optional[int]	,       SCREAMING_SNAKE_CASE_:							Optional[int]	,       SCREAMING_SNAKE_CASE_:							Optional[Any]      )			->      Any:




				'''simple docstring'''





				import requests

				monkeypatch.setattr(SCREAMING_SNAKE_CASE_	,       "request"	,       SCREAMING_SNAKE_CASE_      )

				A__        =			URL
				if issubclass(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
								A__        =			url
				elif issubclass(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
								A__        =			[url]
				elif issubclass(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
								A__        =			{"train": url}
				A__        =			"dummy"
				A__        =			"downloads"
				A__        =			tmp_path
				A__        =			DownloadConfig(
				    cache_dir=os.path.join(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      )	,       use_etag=SCREAMING_SNAKE_CASE_	,       )
				A__        =			DownloadManager(dataset_name=SCREAMING_SNAKE_CASE_	,       download_config=SCREAMING_SNAKE_CASE_      )
				A__        =			dl_manager.download(SCREAMING_SNAKE_CASE_      )
				A__        =			urls
				for downloaded_paths in [downloaded_paths]:
								if isinstance(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
												A__        =			[downloaded_paths]
												A__        =			[urls]
								elif isinstance(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
												assert "train" in downloaded_paths.keys()
												A__        =			downloaded_paths.values()
												A__        =			urls.values()
								assert downloaded_paths
								for downloaded_path, input_url in zip(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
												assert downloaded_path == dl_manager.downloaded_paths[input_url]
												A__        =			Path(SCREAMING_SNAKE_CASE_      )
												A__        =			downloaded_path.parts
												assert parts[-1] == HASH
												assert parts[-2] == cache_subdir
												assert downloaded_path.exists()
												A__        =			downloaded_path.read_text()
												assert content == CONTENT
												A__        =			downloaded_path.with_suffix(".json"      )
												assert metadata_downloaded_path.exists()
												A__        =			json.loads(metadata_downloaded_path.read_text()      )
												assert metadata_content == {"url": URL, "etag": None}



@pytest.mark.parametrize("paths_type"	,       [str, list, dict]      )
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Union[str, Any]	,       SCREAMING_SNAKE_CASE_:							Union[str, Any]	,       SCREAMING_SNAKE_CASE_:							Optional[Any]      )			->      str:




				'''simple docstring'''





				A__        =			str(SCREAMING_SNAKE_CASE_      )
				if issubclass(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
								A__        =			filename
				elif issubclass(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
								A__        =			[filename]
				elif issubclass(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
								A__        =			{"train": filename}
				A__        =			"dummy"
				A__        =			xz_file.parent
				A__        =			"extracted"
				A__        =			DownloadConfig(
				    cache_dir=SCREAMING_SNAKE_CASE_	,       use_etag=SCREAMING_SNAKE_CASE_	,       )
				A__        =			DownloadManager(dataset_name=SCREAMING_SNAKE_CASE_	,       download_config=SCREAMING_SNAKE_CASE_      )
				A__        =			dl_manager.extract(SCREAMING_SNAKE_CASE_      )
				A__        =			paths
				for extracted_paths in [extracted_paths]:
								if isinstance(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
												A__        =			[extracted_paths]
												A__        =			[paths]
								elif isinstance(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
												assert "train" in extracted_paths.keys()
												A__        =			extracted_paths.values()
												A__        =			paths.values()
								assert extracted_paths
								for extracted_path, input_path in zip(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
												assert extracted_path == dl_manager.extracted_paths[input_path]
												A__        =			Path(SCREAMING_SNAKE_CASE_      )
												A__        =			extracted_path.parts
												assert parts[-1] == hash_url_to_filename(SCREAMING_SNAKE_CASE_	,       etag=SCREAMING_SNAKE_CASE_      )
												assert parts[-2] == extracted_subdir
												assert extracted_path.exists()
												A__        =			extracted_path.read_text()
												A__        =			text_file.read_text()
												assert extracted_file_content == expected_file_content



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Optional[Any]	,       SCREAMING_SNAKE_CASE_:							List[Any]      )			->      int:




				'''simple docstring'''





				assert path.endswith(".jsonl"      )
				for num_items, line in enumerate(SCREAMING_SNAKE_CASE_	,       start=1      ):
								A__        =			json.loads(line.decode("utf-8"      )      )
								assert item.keys() == {"col_1", "col_2", "col_3"}
				assert num_items == 4



@pytest.mark.parametrize("archive_jsonl"	,       ["tar_jsonl_path", "zip_jsonl_path"]      )
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Optional[Any]	,       SCREAMING_SNAKE_CASE_:							Union[str, Any]      )			->      Optional[Any]:




				'''simple docstring'''





				A__        =			request.getfixturevalue(SCREAMING_SNAKE_CASE_      )
				A__        =			DownloadManager()
				for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(SCREAMING_SNAKE_CASE_      )	,       start=1      ):
								_test_jsonl(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      )
				assert num_jsonl == 2



@pytest.mark.parametrize("archive_nested_jsonl"	,       ["tar_nested_jsonl_path", "zip_nested_jsonl_path"]      )
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							List[Any]	,       SCREAMING_SNAKE_CASE_:							Optional[int]      )			->      Tuple:




				'''simple docstring'''





				A__        =			request.getfixturevalue(SCREAMING_SNAKE_CASE_      )
				A__        =			DownloadManager()
				for num_tar, (path, file) in enumerate(dl_manager.iter_archive(SCREAMING_SNAKE_CASE_      )	,       start=1      ):
								for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(SCREAMING_SNAKE_CASE_      )	,       start=1      ):
												_test_jsonl(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      )
				assert num_tar == 1
				assert num_jsonl == 2





def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							List[str]      )			->      Any:




				'''simple docstring'''





				A__        =			DownloadManager()
				for num_file, file in enumerate(dl_manager.iter_files(SCREAMING_SNAKE_CASE_      )	,       start=1      ):
								assert os.path.basename(SCREAMING_SNAKE_CASE_      ) == ("test.txt" if num_file == 1 else "train.txt")
				assert num_file == 2




import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
lowerCAmelCase__	   = """src/transformers"""


# This is to make sure the transformers module imported is the one in the repo.
lowerCAmelCase__	   = direct_transformers_import(PATH_TO_TRANSFORMERS)

lowerCAmelCase__	   = transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
lowerCAmelCase__	   = re.compile(R"""\[(.+?)\]\((https://huggingface\.co/.+?)\)""")


lowerCAmelCase__	   = {
    """DecisionTransformerConfig""",
    """EncoderDecoderConfig""",
    """MusicgenConfig""",
    """RagConfig""",
    """SpeechEncoderDecoderConfig""",
    """TimmBackboneConfig""",
    """VisionEncoderDecoderConfig""",
    """VisionTextDualEncoderConfig""",
    """LlamaConfig""",
}



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Union[str, Any]      )			->      Dict:




				'''simple docstring'''





				A__        =			None

				# source code of `config_class`
				A__        =			inspect.getsource(SCREAMING_SNAKE_CASE_      )
				A__        =			_re_checkpoint.findall(SCREAMING_SNAKE_CASE_      )

				# Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
				# For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
				for ckpt_name, ckpt_link in checkpoints:
								# allow the link to end with `/`
								if ckpt_link.endswith("/"      ):
												A__        =			ckpt_link[:-1]

								# verify the checkpoint name corresponds to the checkpoint link
								A__        =			F'https://huggingface.co/{ckpt_name}'
								if ckpt_link == ckpt_link_from_name:
												A__        =			ckpt_name
												break

				return checkpoint



def 					lowerCAmelCase__							(	)			->      List[str]:




				'''simple docstring'''





				A__        =			[]

				for config_class in list(CONFIG_MAPPING.values()      ):
								# Skip deprecated models
								if "models.deprecated" in config_class.__module__:
												continue
								A__        =			get_checkpoint_from_config_class(SCREAMING_SNAKE_CASE_      )

								A__        =			config_class.__name__
								if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
												configs_without_checkpoint.append(SCREAMING_SNAKE_CASE_      )

				if len(SCREAMING_SNAKE_CASE_      ) > 0:
								A__        =			"\n".join(sorted(SCREAMING_SNAKE_CASE_      )      )
								raise ValueError(F'The following configurations don\'t contain any valid checkpoint:\n{message}'      )


if __name__ == "__main__":
	check_config_docstrings_have_checkpoints()

