


import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__  ):
     assert isinstance(UpperCAmelCase__						,							UpperCAmelCase__  )
     assert dataset.num_rows == 4
     assert dataset.num_columns == 3
     assert dataset.column_names == ["col_1", "col_2", "col_3"]
     for feature, expected_dtype in expected_features.items():
          assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""keep_in_memory"""						,							[False, True]  )
def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__  ):
     lowercase_  							=			tmp_path / """cache"""
     lowercase_  							=			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
     with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
          lowercase_  							=			JsonDatasetReader(UpperCAmelCase__						,							cache_dir=UpperCAmelCase__						,							keep_in_memory=UpperCAmelCase__  ).read()
     _check_json_dataset(UpperCAmelCase__						,							UpperCAmelCase__  )


@pytest.mark.parametrize(
    """features"""						,							[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]						,							)
def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__  ):
     lowercase_  							=			tmp_path / """cache"""
     lowercase_  							=			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
     lowercase_  							=			features.copy() if features else default_expected_features
     lowercase_  							=			(
         Features({feature: Value(UpperCAmelCase__  ) for feature, dtype in features.items()}  ) if features is not None else None
     )
     lowercase_  							=			JsonDatasetReader(UpperCAmelCase__						,							features=UpperCAmelCase__						,							cache_dir=UpperCAmelCase__  ).read()
     _check_json_dataset(UpperCAmelCase__						,							UpperCAmelCase__  )


@pytest.mark.parametrize(
    """features"""						,							[
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ]						,							)
def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__  ):
     lowercase_  							=			tmp_path / """cache"""
     lowercase_  							=			{"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""}
     lowercase_  							=			features.copy() if features else default_expected_features
     lowercase_  							=			(
         Features({feature: Value(UpperCAmelCase__  ) for feature, dtype in features.items()}  ) if features is not None else None
     )
     lowercase_  							=			JsonDatasetReader(UpperCAmelCase__						,							features=UpperCAmelCase__						,							cache_dir=UpperCAmelCase__  ).read()
     assert isinstance(UpperCAmelCase__						,							UpperCAmelCase__  )
     assert dataset.num_rows == 2
     assert dataset.num_columns == 3
     assert dataset.column_names == ["col_3", "col_1", "col_2"]
     for feature, expected_dtype in expected_features.items():
          assert dataset.features[feature].dtype == expected_dtype


def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__  ):
     # jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
     lowercase_  							=			{"""col_2""": """int64""", """col_3""": """float64""", """col_1""": """string"""}
     lowercase_  							=			features.copy()
     lowercase_  							=			(
         Features({feature: Value(UpperCAmelCase__  ) for feature, dtype in features.items()}  ) if features is not None else None
     )
     lowercase_  							=			tmp_path / """cache"""
     lowercase_  							=			JsonDatasetReader(UpperCAmelCase__						,							features=UpperCAmelCase__						,							cache_dir=UpperCAmelCase__  ).read()
     assert isinstance(UpperCAmelCase__						,							UpperCAmelCase__  )
     assert dataset.num_rows == 2
     assert dataset.num_columns == 3
     assert dataset.column_names == ["col_2", "col_3", "col_1"]
     for feature, expected_dtype in expected_features.items():
          assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""split"""						,							[None, NamedSplit("""train"""  ), """train""", """test"""]  )
def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__  ):
     lowercase_  							=			tmp_path / """cache"""
     lowercase_  							=			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
     lowercase_  							=			JsonDatasetReader(UpperCAmelCase__						,							cache_dir=UpperCAmelCase__						,							split=UpperCAmelCase__  ).read()
     _check_json_dataset(UpperCAmelCase__						,							UpperCAmelCase__  )
     assert dataset.split == split if split else "train"


@pytest.mark.parametrize("""path_type"""						,							[str, list]  )
def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__  ):
     if issubclass(UpperCAmelCase__						,							UpperCAmelCase__  ):
          lowercase_  							=			jsonl_path
     elif issubclass(UpperCAmelCase__						,							UpperCAmelCase__  ):
          lowercase_  							=			[jsonl_path]
     lowercase_  							=			tmp_path / """cache"""
     lowercase_  							=			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
     lowercase_  							=			JsonDatasetReader(UpperCAmelCase__						,							cache_dir=UpperCAmelCase__  ).read()
     _check_json_dataset(UpperCAmelCase__						,							UpperCAmelCase__  )


def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__=("train",)  ):
     assert isinstance(UpperCAmelCase__						,							UpperCAmelCase__  )
     for split in splits:
          lowercase_  							=			dataset_dict[split]
          assert dataset.num_rows == 4
          assert dataset.num_columns == 3
          assert dataset.column_names == ["col_1", "col_2", "col_3"]
          for feature, expected_dtype in expected_features.items():
               assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""keep_in_memory"""						,							[False, True]  )
def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__  ):
     lowercase_  							=			tmp_path / """cache"""
     lowercase_  							=			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
     with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
          lowercase_  							=			JsonDatasetReader({"""train""": jsonl_path}						,							cache_dir=UpperCAmelCase__						,							keep_in_memory=UpperCAmelCase__  ).read()
     _check_json_datasetdict(UpperCAmelCase__						,							UpperCAmelCase__  )


@pytest.mark.parametrize(
    """features"""						,							[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]						,							)
def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__  ):
     lowercase_  							=			tmp_path / """cache"""
     lowercase_  							=			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
     lowercase_  							=			features.copy() if features else default_expected_features
     lowercase_  							=			(
         Features({feature: Value(UpperCAmelCase__  ) for feature, dtype in features.items()}  ) if features is not None else None
     )
     lowercase_  							=			JsonDatasetReader({"""train""": jsonl_path}						,							features=UpperCAmelCase__						,							cache_dir=UpperCAmelCase__  ).read()
     _check_json_datasetdict(UpperCAmelCase__						,							UpperCAmelCase__  )


@pytest.mark.parametrize("""split"""						,							[None, NamedSplit("""train"""  ), """train""", """test"""]  )
def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__  ):
     if split:
          lowercase_  							=			{split: jsonl_path}
     else:
          lowercase_  							=			"""train"""
          lowercase_  							=			{"""train""": jsonl_path, """test""": jsonl_path}
     lowercase_  							=			tmp_path / """cache"""
     lowercase_  							=			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
     lowercase_  							=			JsonDatasetReader(UpperCAmelCase__						,							cache_dir=UpperCAmelCase__  ).read()
     _check_json_datasetdict(UpperCAmelCase__						,							UpperCAmelCase__						,							splits=list(path.keys()  )  )
     assert all(dataset[split].split == split for split in path.keys()  )


def 			UpperCAmelCase_				(      UpperCAmelCase__  ):
     return json.load(UpperCAmelCase__  )


def 			UpperCAmelCase_				(      UpperCAmelCase__  ):
     return [json.loads(UpperCAmelCase__  ) for line in buffer]







class        UpperCamelCase__   :

     @pytest.mark.parametrize("""lines, load_json_function"""							,					[(True, load_json_lines), (False, load_json)]       )
     def   UpperCAmelCase__       (     self       :   List[Any]							,					UpperCamelCase__       :   Optional[int]							,					UpperCamelCase__       :   Optional[int]							,					UpperCamelCase__       :   Optional[Any]       ):






          '''simple docstring'''






          with io.BytesIO() as buffer:
               JsonDatasetWriter(UpperCamelCase__							,					UpperCamelCase__							,					lines=UpperCamelCase__       ).write()
               buffer.seek(0       )
               lowercase_  							=			load_json_function(UpperCamelCase__       )
          assert isinstance(UpperCamelCase__							,					UpperCamelCase__       )
          assert isinstance(exported_content[0]							,					UpperCamelCase__       )
          assert len(UpperCamelCase__       ) == 10

     @pytest.mark.parametrize(
         """orient, container, keys, len_at"""							,					[
             ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
             ("""split""", dict, {"""columns""", """data"""}, """data"""),
             ("""index""", dict, set("""0123456789"""       ), None),
             ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
             ("""values""", list, None, None),
             ("""table""", dict, {"""schema""", """data"""}, """data"""),
         ]							,					)
     def   UpperCAmelCase__       (     self       :   Any							,					UpperCamelCase__       :   Union[str, Any]							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   Dict							,					UpperCamelCase__       :   Optional[int]							,					UpperCamelCase__       :   int       ):






          '''simple docstring'''






          with io.BytesIO() as buffer:
               JsonDatasetWriter(UpperCamelCase__							,					UpperCamelCase__							,					lines=UpperCamelCase__							,					orient=UpperCamelCase__       ).write()
               buffer.seek(0       )
               lowercase_  							=			load_json(UpperCamelCase__       )
          assert isinstance(UpperCamelCase__							,					UpperCamelCase__       )
          if keys:
               if container is dict:
                    assert exported_content.keys() == keys
               else:
                    assert exported_content[0].keys() == keys
          else:
               assert not hasattr(UpperCamelCase__							,					"""keys"""       ) and not hasattr(exported_content[0]							,					"""keys"""       )
          if len_at:
               assert len(exported_content[len_at]       ) == 10
          else:
               assert len(UpperCamelCase__       ) == 10

     @pytest.mark.parametrize("""lines, load_json_function"""							,					[(True, load_json_lines), (False, load_json)]       )
     def   UpperCAmelCase__       (     self       :   Optional[Any]							,					UpperCamelCase__       :   Dict							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   Dict       ):






          '''simple docstring'''






          with io.BytesIO() as buffer:
               JsonDatasetWriter(UpperCamelCase__							,					UpperCamelCase__							,					lines=UpperCamelCase__							,					num_proc=2       ).write()
               buffer.seek(0       )
               lowercase_  							=			load_json_function(UpperCamelCase__       )
          assert isinstance(UpperCamelCase__							,					UpperCamelCase__       )
          assert isinstance(exported_content[0]							,					UpperCamelCase__       )
          assert len(UpperCamelCase__       ) == 10

     @pytest.mark.parametrize(
         """orient, container, keys, len_at"""							,					[
             ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
             ("""split""", dict, {"""columns""", """data"""}, """data"""),
             ("""index""", dict, set("""0123456789"""       ), None),
             ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
             ("""values""", list, None, None),
             ("""table""", dict, {"""schema""", """data"""}, """data"""),
         ]							,					)
     def   UpperCAmelCase__       (     self       :   Optional[Any]							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   Tuple							,					UpperCamelCase__       :   List[str]							,					UpperCamelCase__       :   Any							,					UpperCamelCase__       :   Optional[int]       ):






          '''simple docstring'''






          with io.BytesIO() as buffer:
               JsonDatasetWriter(UpperCamelCase__							,					UpperCamelCase__							,					lines=UpperCamelCase__							,					orient=UpperCamelCase__							,					num_proc=2       ).write()
               buffer.seek(0       )
               lowercase_  							=			load_json(UpperCamelCase__       )
          assert isinstance(UpperCamelCase__							,					UpperCamelCase__       )
          if keys:
               if container is dict:
                    assert exported_content.keys() == keys
               else:
                    assert exported_content[0].keys() == keys
          else:
               assert not hasattr(UpperCamelCase__							,					"""keys"""       ) and not hasattr(exported_content[0]							,					"""keys"""       )
          if len_at:
               assert len(exported_content[len_at]       ) == 10
          else:
               assert len(UpperCamelCase__       ) == 10

     def   UpperCAmelCase__       (     self       :   Dict							,					UpperCamelCase__       :   Tuple       ):






          '''simple docstring'''






          with pytest.raises(UpperCamelCase__       ):
               with io.BytesIO() as buffer:
                    JsonDatasetWriter(UpperCamelCase__							,					UpperCamelCase__							,					num_proc=0       )







     @pytest.mark.parametrize("""compression, extension"""							,					[("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")]       )
     def   UpperCAmelCase__       (     self       :   Any							,					UpperCamelCase__       :   Optional[Any]							,					UpperCamelCase__       :   List[str]							,					UpperCamelCase__       :   Dict							,					UpperCamelCase__       :   List[Any]							,					UpperCamelCase__       :   Tuple       ):






          '''simple docstring'''






          lowercase_  							=			tmp_path_factory.mktemp("""data"""       ) / F'''test.json.{extension}'''
          lowercase_  							=			str(shared_datadir / F'''test_file.json.{extension}'''       )
          JsonDatasetWriter(UpperCamelCase__							,					UpperCamelCase__							,					compression=UpperCamelCase__       ).write()

          with fsspec.open(UpperCamelCase__							,					"""rb"""							,					compression="""infer"""       ) as f:
               lowercase_  							=			f.read()
          with fsspec.open(UpperCamelCase__							,					"""rb"""							,					compression="""infer"""       ) as f:
               lowercase_  							=			f.read()
          assert exported_content == original_content






import PIL.Image
import PIL.ImageOps
from packaging import version
from PIL import Image


if version.parse(version.parse(PIL.__version__).base_version) >= version.parse('9.1.0'):
							a					  =     {
							    'linear': PIL.Image.Resampling.BILINEAR,
							    'bilinear': PIL.Image.Resampling.BILINEAR,
							    'bicubic': PIL.Image.Resampling.BICUBIC,
							    'lanczos': PIL.Image.Resampling.LANCZOS,
							    'nearest': PIL.Image.Resampling.NEAREST,
							}
else:
							a					  =     {
							    'linear': PIL.Image.LINEAR,
							    'bilinear': PIL.Image.BILINEAR,
							    'bicubic': PIL.Image.BICUBIC,
							    'lanczos': PIL.Image.LANCZOS,
							    'nearest': PIL.Image.NEAREST,
							}


def 			UpperCAmelCase_				(      UpperCAmelCase__  ):
					lowercase_  							=			(images / 2 + 0.5).clamp(0						,							1  )
					lowercase_  							=			images.cpu().permute(0						,							2						,							3						,							1  ).float().numpy()
					lowercase_  							=			numpy_to_pil(UpperCAmelCase__  )
					return images



def 			UpperCAmelCase_				(      UpperCAmelCase__  ):
					if images.ndim == 3:
										lowercase_  							=			images[None, ...]
					lowercase_  							=			(images * 2_5_5).round().astype("""uint8"""  )
					if images.shape[-1] == 1:
										# special case for grayscale (single channel) images
										lowercase_  							=			[Image.fromarray(image.squeeze()						,							mode="""L"""  ) for image in images]
					else:
										lowercase_  							=			[Image.fromarray(UpperCAmelCase__  ) for image in images]

					return pil_images


