





'''simple docstring'''





# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import sys
import warnings
from os.path import abspath, dirname, join


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
lowerCAmelCase_						   =    abspath(join(dirname(dirname(dirname(__file__))), """src"""))
sys.path.insert(1, git_repo_path)


# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action="""ignore""", category=FutureWarning)


def     lowerCamelCase_	(      lowerCAmelCase:	List[Any]				)->		int:
  from transformers.testing_utils import pytest_addoption_shared

  pytest_addoption_shared(lowerCAmelCase				)







def     lowerCamelCase_	(      lowerCAmelCase:	int				)->		str:
  from transformers.testing_utils import pytest_terminal_summary_main

  _snake_case   :				List[Any]														=	terminalreporter.config.getoption('--make-reports'				)
  if make_reports:
    pytest_terminal_summary_main(lowerCAmelCase    ,							id=lowerCAmelCase				)











import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


lowerCAmelCase_						   =    """http://www.mocksite.com/file1.txt"""
lowerCAmelCase_						   =    """\"text\": [\"foo\", \"foo\"]"""
lowerCAmelCase_						   =    """6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8"""



class        _lowerCAmelCase    :


  '''simple docstring'''
  a_							:       int      					=200
  a_							:       List[str]      					={"""Content-Length""": """100"""}
  a_							:       Tuple      					={}







  def UpperCamelCase_ (      self      :					Any      ,				**UpperCamelCase      :					Any							):


    '''simple docstring'''





    return [bytes(UpperCamelCase      ,				'utf-8'							)]




def     lowerCamelCase_	(      *lowerCAmelCase:	Tuple    ,							**lowerCAmelCase:	Tuple				)->		str:
  return MockResponse()


@pytest.mark.parametrize('urls_type'    ,							[str, list, dict]				)
def     lowerCamelCase_	(      lowerCAmelCase:	Union[str, Any]    ,							lowerCAmelCase:	Optional[Any]    ,							lowerCAmelCase:	Dict				)->		Optional[Any]:
  import requests

  monkeypatch.setattr(lowerCAmelCase    ,							'request'    ,							lowerCAmelCase				)

  _snake_case   :				List[str]														=	URL
  if issubclass(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				Optional[int]														=	url
  elif issubclass(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				Any														=	[url]
  elif issubclass(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				Optional[Any]														=	{'train': url}
  _snake_case   :				int														=	'dummy'
  _snake_case   :				Optional[Any]														=	'downloads'
  _snake_case   :				Union[str, Any]														=	tmp_path
  _snake_case   :				Dict														=	DownloadConfig(
      cache_dir=os.path.join(lowerCAmelCase    ,							lowerCAmelCase				)    ,							use_etag=lowerCAmelCase    ,							)
  _snake_case   :				str														=	DownloadManager(dataset_name=lowerCAmelCase    ,							download_config=lowerCAmelCase				)
  _snake_case   :				Optional[int]														=	dl_manager.download(lowerCAmelCase				)
  _snake_case   :				Tuple														=	urls
  for downloaded_paths in [downloaded_paths]:
    if isinstance(lowerCAmelCase    ,							lowerCAmelCase				):
      _snake_case   :				Optional[Any]														=	[downloaded_paths]
      _snake_case   :				List[str]														=	[urls]
    elif isinstance(lowerCAmelCase    ,							lowerCAmelCase				):
      assert "train" in downloaded_paths.keys()
      _snake_case   :				Any														=	downloaded_paths.values()
      _snake_case   :				List[str]														=	urls.values()
    assert downloaded_paths
    for downloaded_path, input_url in zip(lowerCAmelCase    ,							lowerCAmelCase				):
      assert downloaded_path == dl_manager.downloaded_paths[input_url]
      _snake_case   :				str														=	Path(lowerCAmelCase				)
      _snake_case   :				int														=	downloaded_path.parts
      assert parts[-1] == HASH
      assert parts[-2] == cache_subdir
      assert downloaded_path.exists()
      _snake_case   :				List[str]														=	downloaded_path.read_text()
      assert content == CONTENT
      _snake_case   :				Any														=	downloaded_path.with_suffix('.json'				)
      assert metadata_downloaded_path.exists()
      _snake_case   :				Tuple														=	json.loads(metadata_downloaded_path.read_text()				)
      assert metadata_content == {"url": URL, "etag": None}


@pytest.mark.parametrize('paths_type'    ,							[str, list, dict]				)
def     lowerCamelCase_	(      lowerCAmelCase:	Optional[Any]    ,							lowerCAmelCase:	Optional[int]    ,							lowerCAmelCase:	Any				)->		str:
  _snake_case   :				str														=	str(lowerCAmelCase				)
  if issubclass(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				str														=	filename
  elif issubclass(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				List[Any]														=	[filename]
  elif issubclass(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				Optional[Any]														=	{'train': filename}
  _snake_case   :				Any														=	'dummy'
  _snake_case   :				Union[str, Any]														=	xz_file.parent
  _snake_case   :				int														=	'extracted'
  _snake_case   :				Union[str, Any]														=	DownloadConfig(
      cache_dir=lowerCAmelCase    ,							use_etag=lowerCAmelCase    ,							)
  _snake_case   :				List[str]														=	DownloadManager(dataset_name=lowerCAmelCase    ,							download_config=lowerCAmelCase				)
  _snake_case   :				Dict														=	dl_manager.extract(lowerCAmelCase				)
  _snake_case   :				Optional[int]														=	paths
  for extracted_paths in [extracted_paths]:
    if isinstance(lowerCAmelCase    ,							lowerCAmelCase				):
      _snake_case   :				List[str]														=	[extracted_paths]
      _snake_case   :				int														=	[paths]
    elif isinstance(lowerCAmelCase    ,							lowerCAmelCase				):
      assert "train" in extracted_paths.keys()
      _snake_case   :				Optional[int]														=	extracted_paths.values()
      _snake_case   :				str														=	paths.values()
    assert extracted_paths
    for extracted_path, input_path in zip(lowerCAmelCase    ,							lowerCAmelCase				):
      assert extracted_path == dl_manager.extracted_paths[input_path]
      _snake_case   :				List[str]														=	Path(lowerCAmelCase				)
      _snake_case   :				Optional[Any]														=	extracted_path.parts
      assert parts[-1] == hash_url_to_filename(lowerCAmelCase    ,							etag=lowerCAmelCase				)
      assert parts[-2] == extracted_subdir
      assert extracted_path.exists()
      _snake_case   :				Optional[int]														=	extracted_path.read_text()
      _snake_case   :				int														=	text_file.read_text()
      assert extracted_file_content == expected_file_content


def     lowerCamelCase_	(      lowerCAmelCase:	Optional[Any]    ,							lowerCAmelCase:	List[Any]				)->		Dict:
  assert path.endswith('.jsonl'				)
  for num_items, line in enumerate(lowerCAmelCase    ,							start=1				):
    _snake_case   :				Dict														=	json.loads(line.decode('utf-8'				)				)
    assert item.keys() == {"col_1", "col_2", "col_3"}
  assert num_items == 4


@pytest.mark.parametrize('archive_jsonl'    ,							['tar_jsonl_path', 'zip_jsonl_path']				)
def     lowerCamelCase_	(      lowerCAmelCase:	List[Any]    ,							lowerCAmelCase:	List[str]				)->		Dict:
  _snake_case   :				List[str]														=	request.getfixturevalue(lowerCAmelCase				)
  _snake_case   :				Optional[Any]														=	DownloadManager()
  for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(lowerCAmelCase				)    ,							start=1				):
    _test_jsonl(lowerCAmelCase    ,							lowerCAmelCase				)
  assert num_jsonl == 2


@pytest.mark.parametrize('archive_nested_jsonl'    ,							['tar_nested_jsonl_path', 'zip_nested_jsonl_path']				)
def     lowerCamelCase_	(      lowerCAmelCase:	List[Any]    ,							lowerCAmelCase:	int				)->		str:
  _snake_case   :				List[Any]														=	request.getfixturevalue(lowerCAmelCase				)
  _snake_case   :				Optional[int]														=	DownloadManager()
  for num_tar, (path, file) in enumerate(dl_manager.iter_archive(lowerCAmelCase				)    ,							start=1				):
    for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(lowerCAmelCase				)    ,							start=1				):
      _test_jsonl(lowerCAmelCase    ,							lowerCAmelCase				)
  assert num_tar == 1
  assert num_jsonl == 2







def     lowerCamelCase_	(      lowerCAmelCase:	Any				)->		int:
  _snake_case   :				Tuple														=	DownloadManager()
  for num_file, file in enumerate(dl_manager.iter_files(lowerCAmelCase				)    ,							start=1				):
    assert os.path.basename(lowerCAmelCase				) == ("test.txt" if num_file == 1 else "train.txt")
  assert num_file == 2




