"""simple docstring"""




import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_a   :	List[Any] 					=     datasets.utils.logging.get_logger(__name__)
@dataclass
class 		__A							(				datasets.BuilderConfig	):
     _UpperCamelCase							:    Optional[datasets.Features]	      =	None
     _UpperCamelCase							:    str	      =	"utf-8"
     _UpperCamelCase							:    Optional[str]	      =	None
     _UpperCamelCase							:    Optional[str]	      =	None
     _UpperCamelCase							:    bool	      =	True  # deprecated
     _UpperCamelCase							:    Optional[int]	      =	None  # deprecated
     _UpperCamelCase							:    int	      =	10 << 20  # 10MB
     _UpperCamelCase							:    Optional[bool]	      =	None





class 		__A							(				datasets.ArrowBasedBuilder	):
     _UpperCamelCase							:    Union[str, Any]	      =	JsonConfig






     def 					__A						(	self						):
       if self.config.block_size is not None:
         logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""						)
         _lowerCAmelCase		: Optional[Any]						    =  self.config.block_size
       if self.config.use_threads is not True:
         logger.warning(
             """The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore."""						)
       if self.config.newlines_in_values is not None:
         raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""						)
       return datasets.DatasetInfo(features=self.config.features						)






     def 					__A						(	self       ,  a__						):
       if not self.config.data_files:
         raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"						)
       _lowerCAmelCase		: List[str]						    =  dl_manager.download_and_extract(self.config.data_files						)
       if isinstance(lowerCamelCase_       ,  (str, list, tuple)						):
         _lowerCAmelCase		: Tuple						    =  data_files
         if isinstance(lowerCamelCase_       ,  lowerCamelCase_						):
           _lowerCAmelCase		: Optional[Any]						    =  [files]
         _lowerCAmelCase		: Optional[Any]						    =  [dl_manager.iter_files(lowerCamelCase_						) for file in files]
         return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,  gen_kwargs={"""files""": files}						)]
       _lowerCAmelCase		: Tuple						    =  []
       for split_name, files in data_files.items():
         if isinstance(lowerCamelCase_       ,  lowerCamelCase_						):
           _lowerCAmelCase		: Dict						    =  [files]
         _lowerCAmelCase		: Union[str, Any]						    =  [dl_manager.iter_files(lowerCamelCase_						) for file in files]
         splits.append(datasets.SplitGenerator(name=lowerCamelCase_       ,  gen_kwargs={"""files""": files}						)						)
       return splits






     def 					__A						(	self       ,  a__						):
       if self.config.features is not None:
         # adding missing columns
         for column_name in set(self.config.features						) - set(pa_table.column_names						):
           _lowerCAmelCase		: List[str]						    =  self.config.features.arrow_schema.field(lowerCamelCase_						).type
           _lowerCAmelCase		: Dict						    =  pa_table.append_column(lowerCamelCase_       ,  pa.array([None] * len(lowerCamelCase_						)       ,  type=lowerCamelCase_						)						)
         # more expensive cast to support nested structures with keys in a different order
         # allows str <-> int/float or str to Audio for example
         _lowerCAmelCase		: List[str]						    =  table_cast(lowerCamelCase_       ,  self.config.features.arrow_schema						)
       return pa_table






     def 					__A						(	self       ,  a__						):
       for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCamelCase_						)						):
         # If the file is one json object and if we need to look at the list of items in one specific field
         if self.config.field is not None:
           with open(lowerCamelCase_       ,  encoding=self.config.encoding       ,  errors=self.config.encoding_errors						) as f:
             _lowerCAmelCase		: str						    =  json.load(lowerCamelCase_						)

           # We keep only the field we are interested in
           _lowerCAmelCase		: Dict						    =  dataset[self.config.field]

           # We accept two format: a list of dicts or a dict of lists
           if isinstance(lowerCamelCase_       ,  (list, tuple)						):
             _lowerCAmelCase		: Optional[Any]						    =  set().union(*[row.keys() for row in dataset]						)
             _lowerCAmelCase		: str						    =  {col: [row.get(lowerCamelCase_						) for row in dataset] for col in keys}
           else:
             _lowerCAmelCase		: int						    =  dataset
           _lowerCAmelCase		: Any						    =  pa.Table.from_pydict(lowerCamelCase_						)
           yield file_idx, self._cast_table(lowerCamelCase_						)

         # If the file has one json object per line
         else:
           with open(lowerCamelCase_       ,  """rb"""						) as f:
             _lowerCAmelCase		: Union[str, Any]						    =  0
             # Use block_size equal to the chunk size divided by 32 to leverage multithreading
             # Set a default minimum value of 16kB if the chunk size is really small
             _lowerCAmelCase		: Union[str, Any]						    =  max(self.config.chunksize // 32       ,  16 << 10						)
             _lowerCAmelCase		: Tuple						    =  (
                 self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
             )
             while True:
               _lowerCAmelCase		: Optional[Any]						    =  f.read(self.config.chunksize						)
               if not batch:
                 break
               # Finish current line
               try:
                 batch += f.readline()
               except (AttributeError, io.UnsupportedOperation):
                 batch += readline(lowerCamelCase_						)
               # PyArrow only accepts utf-8 encoded bytes
               if self.config.encoding != "utf-8":
                 _lowerCAmelCase		: Any						    =  batch.decode(self.config.encoding       ,  errors=lowerCamelCase_						).encode("""utf-8"""						)
               try:
                 while True:
                   try:
                     _lowerCAmelCase		: Union[str, Any]						    =  paj.read_json(
                         io.BytesIO(lowerCamelCase_						)       ,  read_options=paj.ReadOptions(block_size=lowerCamelCase_						)						)
                     break
                   except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                     if (
                         isinstance(lowerCamelCase_       ,  pa.ArrowInvalid						)
                         and "straddling" not in str(lowerCamelCase_						)
                         or block_size > len(lowerCamelCase_						)
                     ):
                       raise
                     else:
                       # Increase the block size in case it was too small.
                       # The block size will be reset for the next file.
                       logger.debug(
                           F"Batch of {len(lowerCamelCase_						)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."						)
                       block_size *= 2
               except pa.ArrowInvalid as e:
                 try:
                   with open(
                       lowerCamelCase_       ,  encoding=self.config.encoding       ,  errors=self.config.encoding_errors						) as f:
                     _lowerCAmelCase		: str						    =  json.load(lowerCamelCase_						)
                 except json.JSONDecodeError:
                   logger.error(F"Failed to read file '{file}' with error {type(lowerCamelCase_						)}: {e}"						)
                   raise e
                 # If possible, parse the file as a list of json objects and exit the loop
                 if isinstance(lowerCamelCase_       ,  lowerCamelCase_						):  # list is the only sequence type supported in JSON
                   try:
                     _lowerCAmelCase		: List[Any]						    =  set().union(*[row.keys() for row in dataset]						)
                     _lowerCAmelCase		: List[Any]						    =  {col: [row.get(lowerCamelCase_						) for row in dataset] for col in keys}
                     _lowerCAmelCase		: Optional[int]						    =  pa.Table.from_pydict(lowerCamelCase_						)
                   except (pa.ArrowInvalid, AttributeError) as e:
                     logger.error(F"Failed to read file '{file}' with error {type(lowerCamelCase_						)}: {e}"						)
                     raise ValueError(F"Not able to read records in the JSON file at {file}."						) from None
                   yield file_idx, self._cast_table(lowerCamelCase_						)
                   break
                 else:
                   logger.error(F"Failed to read file '{file}' with error {type(lowerCamelCase_						)}: {e}"						)
                   raise ValueError(
                       F"Not able to read records in the JSON file at {file}. "
                       F"You should probably indicate the field of the JSON file containing your records. "
                       F"This JSON file contain the following fields: {str(list(dataset.keys()						)						)}. "
                       F"Select the correct one and provide it as `field='XXX'` to the dataset loading method. "						) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
               yield (file_idx, batch_idx), self._cast_table(lowerCamelCase_						)
               batch_idx += 1





"""simple docstring"""




import pickle
import unittest

import torch

from accelerate import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils import require_cpu
@require_cpu
class 		__A							(				unittest.TestCase	):






     def 					__A						(	self						):
       _lowerCAmelCase		: str						    =  torch.nn.Linear(10       ,  10						)
       _lowerCAmelCase		: Optional[Any]						    =  torch.optim.SGD(model.parameters()       ,  0.1						)
       _lowerCAmelCase		: Optional[Any]						    =  Accelerator()
       _lowerCAmelCase		: Tuple						    =  accelerator.prepare(a__						)
       try:
         pickle.loads(pickle.dumps(a__						)						)
       except Exception as e:
         self.fail(F"Accelerated optimizer pickling failed with {e}"						)
       AcceleratorState._reset_state()




