


'''simple docstring'''







import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


snake_case_     :       Dict						=  datasets.utils.logging.get_logger(__name__)


@dataclass
class 					__a      (datasets.BuilderConfig     ):
							__a					:   Optional[datasets.Features]									=       None
							__a					:   str									=       "utf-8"
							__a					:   Optional[str]									=       None
							__a					:   Optional[str]									=       None
							__a					:   bool									=       True  # deprecated
							__a					:   Optional[int]									=       None  # deprecated
							__a					:   int									=       10 << 20  # 10MB
							__a					:   Optional[bool]									=       None





class 					__a      (datasets.ArrowBasedBuilder     ):
							__a					:   Optional[int]									=       JsonConfig

							def     UpperCAmelCase__	(							self							:     int   )					->       Union[str, Any]:







													"""simple docstring"""
													if self.config.block_size is not None:
																			logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''   )
																			UpperCAmelCase_			:      Optional[int]     						=  self.config.block_size
													if self.config.use_threads is not True:
																			logger.warning(
																			    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''   )
													if self.config.newlines_in_values is not None:
																			raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''   )
													return datasets.DatasetInfo(features=self.config.features   )

							def     UpperCAmelCase__	(							self							:     List[Any]       ,     __magic_name__							:     List[str]   )					->       Tuple:







													"""simple docstring"""
													if not self.config.data_files:
																			raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""   )
													UpperCAmelCase_			:      int     						=  dl_manager.download_and_extract(self.config.data_files   )
													if isinstance(__magic_name__       ,     (str, list, tuple)   ):
																			UpperCAmelCase_			:      Tuple     						=  data_files
																			if isinstance(__magic_name__       ,     __magic_name__   ):
																									UpperCAmelCase_			:      Optional[int]     						=  [files]
																			UpperCAmelCase_			:      Any     						=  [dl_manager.iter_files(__magic_name__   ) for file in files]
																			return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,     gen_kwargs={'''files''': files}   )]
													UpperCAmelCase_			:      Optional[Any]     						=  []
													for split_name, files in data_files.items():
																			if isinstance(__magic_name__       ,     __magic_name__   ):
																									UpperCAmelCase_			:      List[Any]     						=  [files]
																			UpperCAmelCase_			:      Any     						=  [dl_manager.iter_files(__magic_name__   ) for file in files]
																			splits.append(datasets.SplitGenerator(name=__magic_name__       ,     gen_kwargs={'''files''': files}   )   )
													return splits

							def     UpperCAmelCase__	(							self							:     List[str]       ,     __magic_name__							:     pa.Table   )					->       pa.Table:







													"""simple docstring"""
													if self.config.features is not None:
																			# adding missing columns
																			for column_name in set(self.config.features   ) - set(pa_table.column_names   ):
																									UpperCAmelCase_			:      int     						=  self.config.features.arrow_schema.field(__magic_name__   ).type
																									UpperCAmelCase_			:      Optional[int]     						=  pa_table.append_column(__magic_name__       ,     pa.array([None] * len(__magic_name__   )       ,     type=__magic_name__   )   )
																			# more expensive cast to support nested structures with keys in a different order
																			# allows str <-> int/float or str to Audio for example
																			UpperCAmelCase_			:      Optional[Any]     						=  table_cast(__magic_name__       ,     self.config.features.arrow_schema   )
													return pa_table



							def     UpperCAmelCase__	(							self							:     List[Any]       ,     __magic_name__							:     Tuple   )					->       List[Any]:







													"""simple docstring"""
													for file_idx, file in enumerate(itertools.chain.from_iterable(__magic_name__   )   ):
																			# If the file is one json object and if we need to look at the list of items in one specific field
																			if self.config.field is not None:
																									with open(__magic_name__       ,     encoding=self.config.encoding       ,     errors=self.config.encoding_errors   ) as f:
																															UpperCAmelCase_			:      List[str]     						=  json.load(__magic_name__   )

																									# We keep only the field we are interested in
																									UpperCAmelCase_			:      Any     						=  dataset[self.config.field]

																									# We accept two format: a list of dicts or a dict of lists
																									if isinstance(__magic_name__       ,     (list, tuple)   ):
																															UpperCAmelCase_			:      int     						=  set().union(*[row.keys() for row in dataset]   )
																															UpperCAmelCase_			:      List[Any]     						=  {col: [row.get(__magic_name__   ) for row in dataset] for col in keys}
																									else:
																															UpperCAmelCase_			:      Optional[int]     						=  dataset
																									UpperCAmelCase_			:      int     						=  pa.Table.from_pydict(__magic_name__   )
																									yield file_idx, self._cast_table(__magic_name__   )

																			# If the file has one json object per line
																			else:
																									with open(__magic_name__       ,     '''rb'''   ) as f:
																															UpperCAmelCase_			:      Optional[Any]     						=  0
																															# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																															# Set a default minimum value of 16kB if the chunk size is really small
																															UpperCAmelCase_			:      List[Any]     						=  max(self.config.chunksize // 32       ,     16 << 10   )
																															UpperCAmelCase_			:      List[Any]     						=  (
																															    self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
																															)
																															while True:
																																					UpperCAmelCase_			:      List[Any]     						=  f.read(self.config.chunksize   )
																																					if not batch:
																																											break
																																					# Finish current line
																																					try:
																																											batch += f.readline()
																																					except (AttributeError, io.UnsupportedOperation):
																																											batch += readline(__magic_name__   )
																																					# PyArrow only accepts utf-8 encoded bytes
																																					if self.config.encoding != "utf-8":
																																											UpperCAmelCase_			:      List[Any]     						=  batch.decode(self.config.encoding       ,     errors=__magic_name__   ).encode('''utf-8'''   )
																																					try:
																																											while True:
																																																	try:
																																																							UpperCAmelCase_			:      Union[str, Any]     						=  paj.read_json(
																																																							    io.BytesIO(__magic_name__   )       ,     read_options=paj.ReadOptions(block_size=__magic_name__   )   )
																																																							break
																																																	except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																							if (
																																																							    isinstance(__magic_name__       ,     pa.ArrowInvalid   )
																																																							    and "straddling" not in str(__magic_name__   )
																																																							    or block_size > len(__magic_name__   )
																																																							):
																																																													raise
																																																							else:
																																																													# Increase the block size in case it was too small.
																																																													# The block size will be reset for the next file.
																																																													logger.debug(
																																																													    F"""Batch of {len(__magic_name__   )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""   )
																																																													block_size *= 2
																																					except pa.ArrowInvalid as e:
																																											try:
																																																	with open(
																																																	    __magic_name__       ,     encoding=self.config.encoding       ,     errors=self.config.encoding_errors   ) as f:
																																																							UpperCAmelCase_			:      int     						=  json.load(__magic_name__   )
																																											except json.JSONDecodeError:
																																																	logger.error(F"""Failed to read file '{file}' with error {type(__magic_name__   )}: {e}"""   )
																																																	raise e
																																											# If possible, parse the file as a list of json objects and exit the loop
																																											if isinstance(__magic_name__       ,     __magic_name__   ):  # list is the only sequence type supported in JSON
																																																	try:
																																																							UpperCAmelCase_			:      str     						=  set().union(*[row.keys() for row in dataset]   )
																																																							UpperCAmelCase_			:      Optional[int]     						=  {col: [row.get(__magic_name__   ) for row in dataset] for col in keys}
																																																							UpperCAmelCase_			:      Union[str, Any]     						=  pa.Table.from_pydict(__magic_name__   )
																																																	except (pa.ArrowInvalid, AttributeError) as e:
																																																							logger.error(F"""Failed to read file '{file}' with error {type(__magic_name__   )}: {e}"""   )
																																																							raise ValueError(F"""Not able to read records in the JSON file at {file}."""   ) from None
																																																	yield file_idx, self._cast_table(__magic_name__   )
																																																	break
																																											else:
																																																	logger.error(F"""Failed to read file '{file}' with error {type(__magic_name__   )}: {e}"""   )
																																																	raise ValueError(
																																																	    F"""Not able to read records in the JSON file at {file}. """
																																																	    F"""You should probably indicate the field of the JSON file containing your records. """
																																																	    F"""This JSON file contain the following fields: {str(list(dataset.keys()   )   )}. """
																																																	    F"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """   ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																					yield (file_idx, batch_idx), self._cast_table(__magic_name__   )
																																					batch_idx += 1












'''simple docstring'''







class 					__a      :

							def __init__(							self							:     List[Any]       ,     __magic_name__							:     int   )					->       None:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[Any]     						=  size
													UpperCAmelCase_			:      Tuple     						=  [0] * size
													UpperCAmelCase_			:      Optional[Any]     						=  [0] * size

							@staticmethod
							def     UpperCAmelCase__	(							__magic_name__							:     int   )					->       int:







													"""simple docstring"""
													return index | (index + 1)

							@staticmethod
							def     UpperCAmelCase__	(							__magic_name__							:     int   )					->       int:







													"""simple docstring"""
													return (index & (index + 1)) - 1

							def     UpperCAmelCase__	(							self							:     Optional[int]       ,     __magic_name__							:     int       ,     __magic_name__							:     int   )					->       None:







													"""simple docstring"""
													UpperCAmelCase_			:      int     						=  value
													while index < self.size:
																			UpperCAmelCase_			:      str     						=  self.get_prev(__magic_name__   ) + 1
																			if current_left_border == index:
																									UpperCAmelCase_			:      List[str]     						=  value
																			else:
																									UpperCAmelCase_			:      Optional[int]     						=  max(__magic_name__       ,     __magic_name__       ,     __magic_name__   )
																			UpperCAmelCase_			:      Tuple     						=  self.get_next(__magic_name__   )



							def     UpperCAmelCase__	(							self							:     Any       ,     __magic_name__							:     int       ,     __magic_name__							:     int   )					->       int:







													"""simple docstring"""
													right -= 1  # Because of right is exclusive
													UpperCAmelCase_			:      List[str]     						=  0
													while left <= right:
																			UpperCAmelCase_			:      Optional[Any]     						=  self.get_prev(__magic_name__   )
																			if left <= current_left:
																									UpperCAmelCase_			:      Dict     						=  max(__magic_name__       ,     self.tree[right]   )
																									UpperCAmelCase_			:      Optional[Any]     						=  current_left
																			else:
																									UpperCAmelCase_			:      str     						=  max(__magic_name__       ,     self.arr[right]   )
																									right -= 1
													return result


if __name__ == "__main__":
					import doctest

					doctest.testmod()





