




import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def     _SCREAMING_SNAKE_CASE    (							lowercase			: Tuple			,							lowercase			: Optional[Any]  ):


      '''simple docstring'''



      assert isinstance(lowercase			,							lowercase  )
      assert dataset.num_rows == 4
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_1", "col_2", "col_3"]
      for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('keep_in_memory'			,							[False, True]  )
def     _SCREAMING_SNAKE_CASE    (							lowercase			: Any			,							lowercase			: int			,							lowercase			: List[str]  ):


      '''simple docstring'''



      lowerCamelCase_	  =					tmp_path / 'cache'
      lowerCamelCase_	  =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
      with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
            lowerCamelCase_	  =					JsonDatasetReader(lowercase			,							cache_dir=lowercase			,							keep_in_memory=lowercase  ).read()
      _check_json_dataset(lowercase			,							lowercase  )

@pytest.mark.parametrize(
    'features'			,							[
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]			,							)
def     _SCREAMING_SNAKE_CASE    (							lowercase			: Tuple			,							lowercase			: Optional[int]			,							lowercase			: str  ):


      '''simple docstring'''



      lowerCamelCase_	  =					tmp_path / 'cache'
      lowerCamelCase_	  =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
      lowerCamelCase_	  =					features.copy() if features else default_expected_features
      lowerCamelCase_	  =					(
          Features({feature: Value(lowercase  ) for feature, dtype in features.items()}  ) if features is not None else None
      )
      lowerCamelCase_	  =					JsonDatasetReader(lowercase			,							features=lowercase			,							cache_dir=lowercase  ).read()
      _check_json_dataset(lowercase			,							lowercase  )

@pytest.mark.parametrize(
    'features'			,							[
        None,
        {'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'},
    ]			,							)
def     _SCREAMING_SNAKE_CASE    (							lowercase			: List[str]			,							lowercase			: Optional[int]			,							lowercase			: List[Any]  ):


      '''simple docstring'''



      lowerCamelCase_	  =					tmp_path / 'cache'
      lowerCamelCase_	  =					{'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'}
      lowerCamelCase_	  =					features.copy() if features else default_expected_features
      lowerCamelCase_	  =					(
          Features({feature: Value(lowercase  ) for feature, dtype in features.items()}  ) if features is not None else None
      )
      lowerCamelCase_	  =					JsonDatasetReader(lowercase			,							features=lowercase			,							cache_dir=lowercase  ).read()
      assert isinstance(lowercase			,							lowercase  )
      assert dataset.num_rows == 2
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_3", "col_1", "col_2"]
      for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype

def     _SCREAMING_SNAKE_CASE    (							lowercase			: List[Any]			,							lowercase			: Optional[int]  ):


      '''simple docstring'''



      lowerCamelCase_	  =					{'col_2': 'int64', 'col_3': 'float64', 'col_1': 'string'}
      lowerCamelCase_	  =					features.copy()
      lowerCamelCase_	  =					(
          Features({feature: Value(lowercase  ) for feature, dtype in features.items()}  ) if features is not None else None
      )
      lowerCamelCase_	  =					tmp_path / 'cache'
      lowerCamelCase_	  =					JsonDatasetReader(lowercase			,							features=lowercase			,							cache_dir=lowercase  ).read()
      assert isinstance(lowercase			,							lowercase  )
      assert dataset.num_rows == 2
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_2", "col_3", "col_1"]
      for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('split'			,							[None, NamedSplit('train'  ), 'train', 'test']  )
def     _SCREAMING_SNAKE_CASE    (							lowercase			: Optional[int]			,							lowercase			: Tuple			,							lowercase			: List[str]  ):


      '''simple docstring'''



      lowerCamelCase_	  =					tmp_path / 'cache'
      lowerCamelCase_	  =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
      lowerCamelCase_	  =					JsonDatasetReader(lowercase			,							cache_dir=lowercase			,							split=lowercase  ).read()
      _check_json_dataset(lowercase			,							lowercase  )
      assert dataset.split == split if split else "train"

@pytest.mark.parametrize('path_type'			,							[str, list]  )
def     _SCREAMING_SNAKE_CASE    (							lowercase			: str			,							lowercase			: List[str]			,							lowercase			: List[str]  ):


      '''simple docstring'''



      if issubclass(lowercase			,							lowercase  ):
            lowerCamelCase_	  =					jsonl_path
      elif issubclass(lowercase			,							lowercase  ):
            lowerCamelCase_	  =					[jsonl_path]
      lowerCamelCase_	  =					tmp_path / 'cache'
      lowerCamelCase_	  =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
      lowerCamelCase_	  =					JsonDatasetReader(lowercase			,							cache_dir=lowercase  ).read()
      _check_json_dataset(lowercase			,							lowercase  )

def     _SCREAMING_SNAKE_CASE    (							lowercase			: List[Any]			,							lowercase			: int			,							lowercase			: int=("train",)  ):


      '''simple docstring'''



      assert isinstance(lowercase			,							lowercase  )
      for split in splits:
            lowerCamelCase_	  =					dataset_dict[split]
            assert dataset.num_rows == 4
            assert dataset.num_columns == 3
            assert dataset.column_names == ["col_1", "col_2", "col_3"]
            for feature, expected_dtype in expected_features.items():
                  assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('keep_in_memory'			,							[False, True]  )
def     _SCREAMING_SNAKE_CASE    (							lowercase			: int			,							lowercase			: Any			,							lowercase			: Any  ):


      '''simple docstring'''



      lowerCamelCase_	  =					tmp_path / 'cache'
      lowerCamelCase_	  =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
      with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
            lowerCamelCase_	  =					JsonDatasetReader({'train': jsonl_path}			,							cache_dir=lowercase			,							keep_in_memory=lowercase  ).read()
      _check_json_datasetdict(lowercase			,							lowercase  )

@pytest.mark.parametrize(
    'features'			,							[
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]			,							)
def     _SCREAMING_SNAKE_CASE    (							lowercase			: List[str]			,							lowercase			: List[str]			,							lowercase			: Tuple  ):


      '''simple docstring'''



      lowerCamelCase_	  =					tmp_path / 'cache'
      lowerCamelCase_	  =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
      lowerCamelCase_	  =					features.copy() if features else default_expected_features
      lowerCamelCase_	  =					(
          Features({feature: Value(lowercase  ) for feature, dtype in features.items()}  ) if features is not None else None
      )
      lowerCamelCase_	  =					JsonDatasetReader({'train': jsonl_path}			,							features=lowercase			,							cache_dir=lowercase  ).read()
      _check_json_datasetdict(lowercase			,							lowercase  )

@pytest.mark.parametrize('split'			,							[None, NamedSplit('train'  ), 'train', 'test']  )
def     _SCREAMING_SNAKE_CASE    (							lowercase			: Any			,							lowercase			: List[Any]			,							lowercase			: List[str]  ):


      '''simple docstring'''



      if split:
            lowerCamelCase_	  =					{split: jsonl_path}
      else:
            lowerCamelCase_	  =					'train'
            lowerCamelCase_	  =					{'train': jsonl_path, 'test': jsonl_path}
      lowerCamelCase_	  =					tmp_path / 'cache'
      lowerCamelCase_	  =					{'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
      lowerCamelCase_	  =					JsonDatasetReader(lowercase			,							cache_dir=lowercase  ).read()
      _check_json_datasetdict(lowercase			,							lowercase			,							splits=list(path.keys()  )  )
      assert all(dataset[split].split == split for split in path.keys()  )

def     _SCREAMING_SNAKE_CASE    (							lowercase			: List[str]  ):


      '''simple docstring'''



      return json.load(lowercase  )

def     _SCREAMING_SNAKE_CASE    (							lowercase			: int  ):


      '''simple docstring'''



      return [json.loads(lowercase  ) for line in buffer]







class 						A:



 '''simple docstring'''


 @pytest.mark.parametrize('lines, load_json_function'		,							[(True, load_json_lines), (False, load_json)]						)
 def        a__				(       self      :						Optional[Any]		,							A_      :						Dict		,							A_      :						List[Any]		,							A_      :						List[str]						)     ->			Optional[int]:

       """simple docstring"""






       with io.BytesIO() as buffer:
             JsonDatasetWriter(A_		,							A_		,							lines=A_						).write()
             buffer.seek(0						)
             lowerCamelCase_	  =					load_json_function(A_						)
       assert isinstance(A_		,							A_						)
       assert isinstance(exported_content[0]		,							A_						)
       assert len(A_						) == 10

 @pytest.mark.parametrize(
     'orient, container, keys, len_at'		,							[
         ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
         ('split', dict, {'columns', 'data'}, 'data'),
         ('index', dict, set('0123456789'						), None),
         ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
         ('values', list, None, None),
         ('table', dict, {'schema', 'data'}, 'data'),
     ]		,							)
 def        a__				(       self      :						Union[str, Any]		,							A_      :						List[str]		,							A_      :						List[Any]		,							A_      :						List[str]		,							A_      :						Dict		,							A_      :						List[str]						)     ->			List[Any]:

       """simple docstring"""






       with io.BytesIO() as buffer:
             JsonDatasetWriter(A_		,							A_		,							lines=A_		,							orient=A_						).write()
             buffer.seek(0						)
             lowerCamelCase_	  =					load_json(A_						)
       assert isinstance(A_		,							A_						)
       if keys:
             if container is dict:
                   assert exported_content.keys() == keys
             else:
                   assert exported_content[0].keys() == keys
       else:
             assert not hasattr(A_		,							'keys'						) and not hasattr(exported_content[0]		,							'keys'						)
       if len_at:
             assert len(exported_content[len_at]						) == 10
       else:
             assert len(A_						) == 10

 @pytest.mark.parametrize('lines, load_json_function'		,							[(True, load_json_lines), (False, load_json)]						)
 def        a__				(       self      :						str		,							A_      :						Tuple		,							A_      :						List[Any]		,							A_      :						List[str]						)     ->			Optional[int]:

       """simple docstring"""






       with io.BytesIO() as buffer:
             JsonDatasetWriter(A_		,							A_		,							lines=A_		,							num_proc=2						).write()
             buffer.seek(0						)
             lowerCamelCase_	  =					load_json_function(A_						)
       assert isinstance(A_		,							A_						)
       assert isinstance(exported_content[0]		,							A_						)
       assert len(A_						) == 10

 @pytest.mark.parametrize(
     'orient, container, keys, len_at'		,							[
         ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
         ('split', dict, {'columns', 'data'}, 'data'),
         ('index', dict, set('0123456789'						), None),
         ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
         ('values', list, None, None),
         ('table', dict, {'schema', 'data'}, 'data'),
     ]		,							)
 def        a__				(       self      :						str		,							A_      :						int		,							A_      :						Union[str, Any]		,							A_      :						Tuple		,							A_      :						int		,							A_      :						Union[str, Any]						)     ->			Optional[int]:

       """simple docstring"""






       with io.BytesIO() as buffer:
             JsonDatasetWriter(A_		,							A_		,							lines=A_		,							orient=A_		,							num_proc=2						).write()
             buffer.seek(0						)
             lowerCamelCase_	  =					load_json(A_						)
       assert isinstance(A_		,							A_						)
       if keys:
             if container is dict:
                   assert exported_content.keys() == keys
             else:
                   assert exported_content[0].keys() == keys
       else:
             assert not hasattr(A_		,							'keys'						) and not hasattr(exported_content[0]		,							'keys'						)
       if len_at:
             assert len(exported_content[len_at]						) == 10
       else:
             assert len(A_						) == 10

 def        a__				(       self      :						Dict		,							A_      :						str						)     ->			List[str]:

       """simple docstring"""






       with pytest.raises(A_						):
             with io.BytesIO() as buffer:
                   JsonDatasetWriter(A_		,							A_		,							num_proc=0						)





 @pytest.mark.parametrize('compression, extension'		,							[('gzip', 'gz'), ('bz2', 'bz2'), ('xz', 'xz')]						)
 def        a__				(       self      :						Any		,							A_      :						Union[str, Any]		,							A_      :						int		,							A_      :						Tuple		,							A_      :						Union[str, Any]		,							A_      :						Optional[int]						)     ->			List[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					tmp_path_factory.mktemp('data'						) / f"""test.json.{extension}"""
       lowerCamelCase_	  =					str(shared_datadir / f"""test_file.json.{extension}"""						)
       JsonDatasetWriter(A_		,							A_		,							compression=A_						).write()

       with fsspec.open(A_		,							'rb'		,							compression='infer'						) as f:
             lowerCamelCase_	  =					f.read()
       with fsspec.open(A_		,							'rb'		,							compression='infer'						) as f:
             lowerCamelCase_	  =					f.read()
       assert exported_content == original_content









from manim import *







class 						A(		UpperCamelCase     ):



 '''simple docstring'''


 def        a__				(       self      :						Optional[Any]						)     ->			List[str]:

       """simple docstring"""






       lowerCamelCase_	  =					Rectangle(height=0.5		,							width=0.5						)
       lowerCamelCase_	  =					Rectangle(height=0.46		,							width=0.46						).set_stroke(width=0						)

       lowerCamelCase_	  =					Rectangle(height=0.25		,							width=0.25						)

       lowerCamelCase_	  =					[mem.copy() for i in range(6						)]
       lowerCamelCase_	  =					[mem.copy() for i in range(6						)]
       lowerCamelCase_	  =					VGroup(*A_						).arrange(A_		,							buff=0						)
       lowerCamelCase_	  =					VGroup(*A_						).arrange(A_		,							buff=0						)
       lowerCamelCase_	  =					VGroup(A_		,							A_						).arrange(A_		,							buff=0						)
       lowerCamelCase_	  =					Text('CPU'		,							font_size=24						)
       lowerCamelCase_	  =					Group(A_		,							A_						).arrange(A_		,							buff=0.5		,							aligned_edge=A_						)
       cpu.move_to([-2.5, -0.5, 0]						)
       self.add(A_						)

       lowerCamelCase_	  =					[mem.copy() for i in range(4						)]
       lowerCamelCase_	  =					VGroup(*A_						).arrange(A_		,							buff=0						)
       lowerCamelCase_	  =					Text('GPU'		,							font_size=24						)
       lowerCamelCase_	  =					Group(A_		,							A_						).arrange(A_		,							buff=0.5		,							aligned_edge=A_						)
       gpu.move_to([-1, -1, 0]						)
       self.add(A_						)

       lowerCamelCase_	  =					[mem.copy() for i in range(6						)]
       lowerCamelCase_	  =					VGroup(*A_						).arrange(A_		,							buff=0						)

       lowerCamelCase_	  =					Text('Model'		,							font_size=24						)
       lowerCamelCase_	  =					Group(A_		,							A_						).arrange(A_		,							buff=0.5		,							aligned_edge=A_						)
       model.move_to([3, -1.0, 0]						)
       self.add(A_						)

       lowerCamelCase_	  =					[]
       lowerCamelCase_	  =					[]

       for i, rect in enumerate(A_						):
             lowerCamelCase_	  =					fill.copy().set_fill(A_		,							opacity=0.8						)
             target.move_to(A_						)
             model_arr.append(A_						)

             lowerCamelCase_	  =					Rectangle(height=0.46		,							width=0.46						).set_stroke(width=0.0						).set_fill(A_		,							opacity=0.8						)
             cpu_target.move_to(cpu_left_col_base[i]						)
             model_cpu_arr.append(A_						)

       self.add(*A_		,							*A_						)

       lowerCamelCase_	  =					[meta_mem.copy() for i in range(6						)]
       lowerCamelCase_	  =					[meta_mem.copy() for i in range(6						)]
       lowerCamelCase_	  =					VGroup(*A_						).arrange(A_		,							buff=0						)
       lowerCamelCase_	  =					VGroup(*A_						).arrange(A_		,							buff=0						)
       lowerCamelCase_	  =					VGroup(A_		,							A_						).arrange(A_		,							buff=0						)
       lowerCamelCase_	  =					Text('Disk'		,							font_size=24						)
       lowerCamelCase_	  =					Group(A_		,							A_						).arrange(A_		,							buff=0.5		,							aligned_edge=A_						)
       disk.move_to([-4, -1.25, 0]						)
       self.add(A_		,							A_						)

       lowerCamelCase_	  =					Square(side_length=2.2						)
       key.move_to([-5, 2, 0]						)

       lowerCamelCase_	  =					MarkupText(
           f"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"""		,							font_size=18		,							)

       key_text.move_to([-5, 2.4, 0]						)

       self.add(A_		,							A_						)

       lowerCamelCase_	  =					MarkupText(
           f"""<span fgcolor='{BLUE}'>●</span> Checkpoint"""		,							font_size=18		,							)

       blue_text.next_to(A_		,							DOWN * 2.4		,							aligned_edge=key_text.get_left()						)
       self.add(A_						)

       lowerCamelCase_	  =					MarkupText(
           f"""Now watch as an input is passed through the model\nand how the memory is utilized and handled."""		,							font_size=24		,							)
       step_a.move_to([2, 2, 0]						)

       self.play(Write(A_						)						)

       lowerCamelCase_	  =					Square(0.3						)
       input.set_fill(A_		,							opacity=1.0						)
       input.set_stroke(width=0.0						)
       input.next_to(model_base[0]		,							A_		,							buff=0.5						)

       self.play(Write(A_						)						)

       input.generate_target()
       input.target.next_to(model_arr[0]		,							direction=A_		,							buff=0.02						)
       self.play(MoveToTarget(A_						)						)

       self.play(FadeOut(A_						)						)

       lowerCamelCase_	  =					Arrow(start=A_		,							end=A_		,							color=A_		,							buff=0.5						)
       a.next_to(model_arr[0].get_left()		,							A_		,							buff=0.2						)

       model_cpu_arr[0].generate_target()
       model_cpu_arr[0].target.move_to(gpu_rect[0]						)

       lowerCamelCase_	  =					MarkupText(
           f"""As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back."""		,							font_size=24		,							)
       step_a.move_to([2, 2, 0]						)

       self.play(Write(A_		,							run_time=3						)						)

       lowerCamelCase_	  =					{'run_time': 1, 'fade_in': True, 'fade_out': True, 'buff': 0.02}

       self.play(
           Write(A_						)		,							Circumscribe(model_arr[0]		,							color=A_		,							**A_						)		,							Circumscribe(model_cpu_arr[0]		,							color=A_		,							**A_						)		,							Circumscribe(gpu_rect[0]		,							color=A_		,							**A_						)		,							)
       self.play(MoveToTarget(model_cpu_arr[0]						)						)

       lowerCamelCase_	  =					a.copy()
       for i in range(6						):
             a_c.next_to(model_arr[i].get_right() + 0.02		,							A_		,							buff=0.2						)

             input.generate_target()
             input.target.move_to(model_arr[i].get_right() + 0.02						)

             lowerCamelCase_	  =					AnimationGroup(
                 FadeOut(A_		,							run_time=0.5						)		,							MoveToTarget(A_		,							run_time=0.5						)		,							FadeIn(A_		,							run_time=0.5						)		,							lag_ratio=0.2						)

             self.play(A_						)

             model_cpu_arr[i].generate_target()
             model_cpu_arr[i].target.move_to(cpu_left_col_base[i]						)

             if i < 5:
                   model_cpu_arr[i + 1].generate_target()
                   model_cpu_arr[i + 1].target.move_to(gpu_rect[0]						)
                   if i >= 1:
                         lowerCamelCase_	  =					0.7

                   self.play(
                       Circumscribe(model_arr[i]		,							**A_						)		,							Circumscribe(cpu_left_col_base[i]		,							**A_						)		,							Circumscribe(cpu_left_col_base[i + 1]		,							color=A_		,							**A_						)		,							Circumscribe(gpu_rect[0]		,							color=A_		,							**A_						)		,							Circumscribe(model_arr[i + 1]		,							color=A_		,							**A_						)		,							)
                   if i < 1:
                         self.play(
                             MoveToTarget(model_cpu_arr[i]						)		,							MoveToTarget(model_cpu_arr[i + 1]						)		,							)
                   else:
                         self.play(
                             MoveToTarget(model_cpu_arr[i]		,							run_time=0.7						)		,							MoveToTarget(model_cpu_arr[i + 1]		,							run_time=0.7						)		,							)
             else:
                   model_cpu_arr[i].generate_target()
                   model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]						)
                   input.generate_target()
                   input.target.next_to(model_arr[-1].get_right()		,							RIGHT + 0.02		,							buff=0.2						)

                   self.play(
                       Circumscribe(model_arr[-1]		,							color=A_		,							**A_						)		,							Circumscribe(cpu_left_col_base[-1]		,							color=A_		,							**A_						)		,							Circumscribe(gpu_rect[0]		,							color=A_		,							**A_						)		,							)

                   self.play(MoveToTarget(model_cpu_arr[i]						)						)

             lowerCamelCase_	  =					a_c
             lowerCamelCase_	  =					a_c.copy()

       input.generate_target()
       input.target.next_to(model_base[-1]		,							RIGHT + 0.02		,							buff=0.5						)
       self.play(
           FadeOut(A_						)		,							FadeOut(A_		,							run_time=0.5						)		,							)

       lowerCamelCase_	  =					MarkupText(f"""Inference on a model too large for GPU memory\nis successfully completed."""		,							font_size=24						)
       step_a.move_to([2, 2, 0]						)

       self.play(Write(A_		,							run_time=3						)		,							MoveToTarget(A_						)						)

       self.wait()



