

import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							str	,       SCREAMING_SNAKE_CASE_:							Tuple      )			->      Optional[int]:




				'''simple docstring'''





				assert isinstance(_A	,       _A      )
				assert dataset.num_rows == 4
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_1", "col_2", "col_3"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype



@pytest.mark.parametrize("keep_in_memory"	,       [False, True]      )
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Union[str, Any]	,       SCREAMING_SNAKE_CASE_:							Any	,       SCREAMING_SNAKE_CASE_:							Optional[int]      )			->      Optional[int]:




				'''simple docstring'''





				A__        =			tmp_path / "cache"
				A__        =			{"col_1": "string", "col_2": "int64", "col_3": "float64"}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								A__        =			JsonDatasetReader(_A	,       cache_dir=_A	,       keep_in_memory=_A      ).read()
				_check_json_dataset(_A	,       _A      )



@pytest.mark.parametrize(
    "features"	,       [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]	,       )
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Tuple	,       SCREAMING_SNAKE_CASE_:							Union[str, Any]	,       SCREAMING_SNAKE_CASE_:							Any      )			->      int:




				'''simple docstring'''





				A__        =			tmp_path / "cache"
				A__        =			{"col_1": "string", "col_2": "int64", "col_3": "float64"}
				A__        =			features.copy() if features else default_expected_features
				A__        =			(
				    Features({feature: Value(_A      ) for feature, dtype in features.items()}      ) if features is not None else None
				)
				A__        =			JsonDatasetReader(_A	,       features=_A	,       cache_dir=_A      ).read()
				_check_json_dataset(_A	,       _A      )



@pytest.mark.parametrize(
    "features"	,       [
        None,
        {"col_3": "float64", "col_1": "string", "col_2": "int64"},
    ]	,       )
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Tuple	,       SCREAMING_SNAKE_CASE_:							Optional[Any]	,       SCREAMING_SNAKE_CASE_:							List[Any]      )			->      Dict:




				'''simple docstring'''





				A__        =			tmp_path / "cache"
				A__        =			{"col_3": "float64", "col_1": "string", "col_2": "int64"}
				A__        =			features.copy() if features else default_expected_features
				A__        =			(
				    Features({feature: Value(_A      ) for feature, dtype in features.items()}      ) if features is not None else None
				)
				A__        =			JsonDatasetReader(_A	,       features=_A	,       cache_dir=_A      ).read()
				assert isinstance(_A	,       _A      )
				assert dataset.num_rows == 2
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_3", "col_1", "col_2"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Union[str, Any]	,       SCREAMING_SNAKE_CASE_:							Optional[Any]      )			->      Dict:




				'''simple docstring'''





				A__        =			{"col_2": "int64", "col_3": "float64", "col_1": "string"}
				A__        =			features.copy()
				A__        =			(
				    Features({feature: Value(_A      ) for feature, dtype in features.items()}      ) if features is not None else None
				)
				A__        =			tmp_path / "cache"
				A__        =			JsonDatasetReader(_A	,       features=_A	,       cache_dir=_A      ).read()
				assert isinstance(_A	,       _A      )
				assert dataset.num_rows == 2
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_2", "col_3", "col_1"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype



@pytest.mark.parametrize("split"	,       [None, NamedSplit("train"      ), "train", "test"]      )
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							int	,       SCREAMING_SNAKE_CASE_:							Any	,       SCREAMING_SNAKE_CASE_:							Tuple      )			->      Optional[int]:




				'''simple docstring'''





				A__        =			tmp_path / "cache"
				A__        =			{"col_1": "string", "col_2": "int64", "col_3": "float64"}
				A__        =			JsonDatasetReader(_A	,       cache_dir=_A	,       split=_A      ).read()
				_check_json_dataset(_A	,       _A      )
				assert dataset.split == split if split else "train"



@pytest.mark.parametrize("path_type"	,       [str, list]      )
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							List[Any]	,       SCREAMING_SNAKE_CASE_:							Tuple	,       SCREAMING_SNAKE_CASE_:							Union[str, Any]      )			->      Union[str, Any]:




				'''simple docstring'''





				if issubclass(_A	,       _A      ):
								A__        =			jsonl_path
				elif issubclass(_A	,       _A      ):
								A__        =			[jsonl_path]
				A__        =			tmp_path / "cache"
				A__        =			{"col_1": "string", "col_2": "int64", "col_3": "float64"}
				A__        =			JsonDatasetReader(_A	,       cache_dir=_A      ).read()
				_check_json_dataset(_A	,       _A      )



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Tuple	,       SCREAMING_SNAKE_CASE_:							List[Any]	,       SCREAMING_SNAKE_CASE_:							int=("train",)      )			->      Optional[Any]:




				'''simple docstring'''





				assert isinstance(_A	,       _A      )
				for split in splits:
								A__        =			dataset_dict[split]
								assert dataset.num_rows == 4
								assert dataset.num_columns == 3
								assert dataset.column_names == ["col_1", "col_2", "col_3"]
								for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype



@pytest.mark.parametrize("keep_in_memory"	,       [False, True]      )
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Any	,       SCREAMING_SNAKE_CASE_:							List[Any]	,       SCREAMING_SNAKE_CASE_:							Optional[int]      )			->      int:




				'''simple docstring'''





				A__        =			tmp_path / "cache"
				A__        =			{"col_1": "string", "col_2": "int64", "col_3": "float64"}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								A__        =			JsonDatasetReader({"train": jsonl_path}	,       cache_dir=_A	,       keep_in_memory=_A      ).read()
				_check_json_datasetdict(_A	,       _A      )



@pytest.mark.parametrize(
    "features"	,       [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]	,       )
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							str	,       SCREAMING_SNAKE_CASE_:							List[str]	,       SCREAMING_SNAKE_CASE_:							str      )			->      List[Any]:




				'''simple docstring'''





				A__        =			tmp_path / "cache"
				A__        =			{"col_1": "string", "col_2": "int64", "col_3": "float64"}
				A__        =			features.copy() if features else default_expected_features
				A__        =			(
				    Features({feature: Value(_A      ) for feature, dtype in features.items()}      ) if features is not None else None
				)
				A__        =			JsonDatasetReader({"train": jsonl_path}	,       features=_A	,       cache_dir=_A      ).read()
				_check_json_datasetdict(_A	,       _A      )



@pytest.mark.parametrize("split"	,       [None, NamedSplit("train"      ), "train", "test"]      )
def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Any	,       SCREAMING_SNAKE_CASE_:							Optional[Any]	,       SCREAMING_SNAKE_CASE_:							List[Any]      )			->      Optional[int]:




				'''simple docstring'''





				if split:
								A__        =			{split: jsonl_path}
				else:
								A__        =			"train"
								A__        =			{"train": jsonl_path, "test": jsonl_path}
				A__        =			tmp_path / "cache"
				A__        =			{"col_1": "string", "col_2": "int64", "col_3": "float64"}
				A__        =			JsonDatasetReader(_A	,       cache_dir=_A      ).read()
				_check_json_datasetdict(_A	,       _A	,       splits=list(path.keys()      )      )
				assert all(dataset[split].split == split for split in path.keys()      )



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Optional[int]      )			->      int:




				'''simple docstring'''





				return json.load(_A      )



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Optional[Any]      )			->      Dict:




				'''simple docstring'''





				return [json.loads(_A      ) for line in buffer]





class 					a__						:
			"""simple docstring"""







			@pytest.mark.parametrize("lines, load_json_function"      ,      [(True, load_json_lines), (False, load_json)]       )
			def      UpperCamelCase		(    self      ,      lowercase      ,      lowercase      ,      lowercase       )  ->						str:
							'''simple docstring'''






							with io.BytesIO() as buffer:
											JsonDatasetWriter(__UpperCamelCase      ,      __UpperCamelCase      ,      lines=__UpperCamelCase       ).write()
											buffer.seek(0       )
											A__        =			load_json_function(__UpperCamelCase       )
							assert isinstance(__UpperCamelCase      ,      __UpperCamelCase       )
							assert isinstance(exported_content[0]      ,      __UpperCamelCase       )
							assert len(__UpperCamelCase       ) == 10


			@pytest.mark.parametrize(
			    "orient, container, keys, len_at"      ,      [
			        ("records", list, {"tokens", "labels", "answers", "id"}, None),
			        ("split", dict, {"columns", "data"}, "data"),
			        ("index", dict, set("0123456789"       ), None),
			        ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
			        ("values", list, None, None),
			        ("table", dict, {"schema", "data"}, "data"),
			    ]      ,      )
			def      UpperCamelCase		(    self      ,      lowercase      ,      lowercase      ,      lowercase      ,      lowercase      ,      lowercase       )  ->						Optional[int]:
							'''simple docstring'''






							with io.BytesIO() as buffer:
											JsonDatasetWriter(__UpperCamelCase      ,      __UpperCamelCase      ,      lines=__UpperCamelCase      ,      orient=__UpperCamelCase       ).write()
											buffer.seek(0       )
											A__        =			load_json(__UpperCamelCase       )
							assert isinstance(__UpperCamelCase      ,      __UpperCamelCase       )
							if keys:
											if container is dict:
															assert exported_content.keys() == keys
											else:
															assert exported_content[0].keys() == keys
							else:
											assert not hasattr(__UpperCamelCase      ,      "keys"       ) and not hasattr(exported_content[0]      ,      "keys"       )
							if len_at:
											assert len(exported_content[len_at]       ) == 10
							else:
											assert len(__UpperCamelCase       ) == 10


			@pytest.mark.parametrize("lines, load_json_function"      ,      [(True, load_json_lines), (False, load_json)]       )
			def      UpperCamelCase		(    self      ,      lowercase      ,      lowercase      ,      lowercase       )  ->						Union[str, Any]:
							'''simple docstring'''






							with io.BytesIO() as buffer:
											JsonDatasetWriter(__UpperCamelCase      ,      __UpperCamelCase      ,      lines=__UpperCamelCase      ,      num_proc=2       ).write()
											buffer.seek(0       )
											A__        =			load_json_function(__UpperCamelCase       )
							assert isinstance(__UpperCamelCase      ,      __UpperCamelCase       )
							assert isinstance(exported_content[0]      ,      __UpperCamelCase       )
							assert len(__UpperCamelCase       ) == 10


			@pytest.mark.parametrize(
			    "orient, container, keys, len_at"      ,      [
			        ("records", list, {"tokens", "labels", "answers", "id"}, None),
			        ("split", dict, {"columns", "data"}, "data"),
			        ("index", dict, set("0123456789"       ), None),
			        ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
			        ("values", list, None, None),
			        ("table", dict, {"schema", "data"}, "data"),
			    ]      ,      )
			def      UpperCamelCase		(    self      ,      lowercase      ,      lowercase      ,      lowercase      ,      lowercase      ,      lowercase       )  ->						List[str]:
							'''simple docstring'''






							with io.BytesIO() as buffer:
											JsonDatasetWriter(__UpperCamelCase      ,      __UpperCamelCase      ,      lines=__UpperCamelCase      ,      orient=__UpperCamelCase      ,      num_proc=2       ).write()
											buffer.seek(0       )
											A__        =			load_json(__UpperCamelCase       )
							assert isinstance(__UpperCamelCase      ,      __UpperCamelCase       )
							if keys:
											if container is dict:
															assert exported_content.keys() == keys
											else:
															assert exported_content[0].keys() == keys
							else:
											assert not hasattr(__UpperCamelCase      ,      "keys"       ) and not hasattr(exported_content[0]      ,      "keys"       )
							if len_at:
											assert len(exported_content[len_at]       ) == 10
							else:
											assert len(__UpperCamelCase       ) == 10


			def      UpperCamelCase		(    self      ,      lowercase       )  ->						List[Any]:
							'''simple docstring'''






							with pytest.raises(__UpperCamelCase       ):
											with io.BytesIO() as buffer:
															JsonDatasetWriter(__UpperCamelCase      ,      __UpperCamelCase      ,      num_proc=0       )


			@pytest.mark.parametrize("compression, extension"      ,      [("gzip", "gz"), ("bz2", "bz2"), ("xz", "xz")]       )
			def      UpperCamelCase		(    self      ,      lowercase      ,      lowercase      ,      lowercase      ,      lowercase      ,      lowercase       )  ->						Dict:
							'''simple docstring'''






							A__        =			tmp_path_factory.mktemp("data"       ) / F'test.json.{extension}'
							A__        =			str(shared_datadir / F'test_file.json.{extension}'       )
							JsonDatasetWriter(__UpperCamelCase      ,      __UpperCamelCase      ,      compression=__UpperCamelCase       ).write()

							with fsspec.open(__UpperCamelCase      ,      "rb"      ,      compression="infer"       ) as f:
											A__        =			f.read()
							with fsspec.open(__UpperCamelCase      ,      "rb"      ,      compression="infer"       ) as f:
											A__        =			f.read()
							assert exported_content == original_content




import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
lowerCAmelCase__	   = """src/transformers"""


# This is to make sure the transformers module imported is the one in the repo.
lowerCAmelCase__	   = direct_transformers_import(PATH_TO_TRANSFORMERS)

lowerCAmelCase__	   = transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
lowerCAmelCase__	   = re.compile(R"""\[(.+?)\]\((https://huggingface\.co/.+?)\)""")


lowerCAmelCase__	   = {
    """DecisionTransformerConfig""",
    """EncoderDecoderConfig""",
    """MusicgenConfig""",
    """RagConfig""",
    """SpeechEncoderDecoderConfig""",
    """TimmBackboneConfig""",
    """VisionEncoderDecoderConfig""",
    """VisionTextDualEncoderConfig""",
    """LlamaConfig""",
}



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Union[str, Any]      )			->      Dict:




				'''simple docstring'''





				A__        =			None

				# source code of `config_class`
				A__        =			inspect.getsource(SCREAMING_SNAKE_CASE_      )
				A__        =			_re_checkpoint.findall(SCREAMING_SNAKE_CASE_      )

				# Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
				# For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
				for ckpt_name, ckpt_link in checkpoints:
								# allow the link to end with `/`
								if ckpt_link.endswith("/"      ):
												A__        =			ckpt_link[:-1]

								# verify the checkpoint name corresponds to the checkpoint link
								A__        =			F'https://huggingface.co/{ckpt_name}'
								if ckpt_link == ckpt_link_from_name:
												A__        =			ckpt_name
												break

				return checkpoint



def 					lowerCAmelCase__							(	)			->      List[str]:




				'''simple docstring'''





				A__        =			[]

				for config_class in list(CONFIG_MAPPING.values()      ):
								# Skip deprecated models
								if "models.deprecated" in config_class.__module__:
												continue
								A__        =			get_checkpoint_from_config_class(SCREAMING_SNAKE_CASE_      )

								A__        =			config_class.__name__
								if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
												configs_without_checkpoint.append(SCREAMING_SNAKE_CASE_      )

				if len(SCREAMING_SNAKE_CASE_      ) > 0:
								A__        =			"\n".join(sorted(SCREAMING_SNAKE_CASE_      )      )
								raise ValueError(F'The following configurations don\'t contain any valid checkpoint:\n{message}'      )


if __name__ == "__main__":
	check_config_docstrings_have_checkpoints()

