


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowerCAmelCase							=						datasets.utils.logging.get_logger(__name__)




@dataclass
class        lowerCamelCase      (     datasets.BuilderConfig   ):
	_lowerCAmelCase    :		Optional[datasets.Features]        =	None
	_lowerCAmelCase    :		str        =	"utf-8"
	_lowerCAmelCase    :		Optional[str]        =	None
	_lowerCAmelCase    :		Optional[str]        =	None
	_lowerCAmelCase    :		bool        =	True  # deprecated
	_lowerCAmelCase    :		Optional[int]        =	None  # deprecated
	_lowerCAmelCase    :		int        =	1_0 << 2_0  # 10MB
	_lowerCAmelCase    :		Optional[bool]        =	None






class        lowerCamelCase      (     datasets.ArrowBasedBuilder   ):
	_lowerCAmelCase    :		Optional[Any]        =	JsonConfig


	def    A(						self):
							if self.config.block_size is not None:
													logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead''')
													__UpperCAmelCase :       Optional[Any]  						=  self.config.block_size
							if self.config.use_threads is not True:
													logger.warning(
													    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.''')
							if self.config.newlines_in_values is not None:
													raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported''')
							return datasets.DatasetInfo(features=self.config.features)


	def    A(						self							,    lowercase__):
							if not self.config.data_files:
													raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}")
							__UpperCAmelCase :       Optional[int]  						=  dl_manager.download_and_extract(self.config.data_files)
							if isinstance(lowercase__							,    (str, list, tuple)):
													__UpperCAmelCase :       Any  						=  data_files
													if isinstance(lowercase__							,    lowercase__):
																			__UpperCAmelCase :       List[str]  						=  [files]
													__UpperCAmelCase :       str  						=  [dl_manager.iter_files(lowercase__) for file in files]
													return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,    gen_kwargs={'''files''': files})]
							__UpperCAmelCase :       int  						=  []
							for split_name, files in data_files.items():
													if isinstance(lowercase__							,    lowercase__):
																			__UpperCAmelCase :       str  						=  [files]
													__UpperCAmelCase :       Dict  						=  [dl_manager.iter_files(lowercase__) for file in files]
													splits.append(datasets.SplitGenerator(name=lowercase__							,    gen_kwargs={'''files''': files}))
							return splits


	def    A(						self							,    lowercase__):
							if self.config.features is not None:
													# adding missing columns
													for column_name in set(self.config.features) - set(pa_table.column_names):
																			__UpperCAmelCase :       Optional[int]  						=  self.config.features.arrow_schema.field(lowercase__).type
																			__UpperCAmelCase :       List[Any]  						=  pa_table.append_column(lowercase__							,    pa.array([None] * len(lowercase__)							,    type=lowercase__))
													# more expensive cast to support nested structures with keys in a different order
													# allows str <-> int/float or str to Audio for example
													__UpperCAmelCase :       str  						=  table_cast(lowercase__							,    self.config.features.arrow_schema)
							return pa_table


	def    A(						self							,    lowercase__):
							for file_idx, file in enumerate(itertools.chain.from_iterable(lowercase__)):
													# If the file is one json object and if we need to look at the list of items in one specific field
													if self.config.field is not None:
																			with open(lowercase__							,    encoding=self.config.encoding							,    errors=self.config.encoding_errors) as f:
																									__UpperCAmelCase :       int  						=  json.load(lowercase__)

																			# We keep only the field we are interested in
																			__UpperCAmelCase :       List[Any]  						=  dataset[self.config.field]

																			# We accept two format: a list of dicts or a dict of lists
																			if isinstance(lowercase__							,    (list, tuple)):
																									__UpperCAmelCase :       Any  						=  set().union(*[row.keys() for row in dataset])
																									__UpperCAmelCase :       str  						=  {col: [row.get(lowercase__) for row in dataset] for col in keys}
																			else:
																									__UpperCAmelCase :       List[str]  						=  dataset
																			__UpperCAmelCase :       Any  						=  pa.Table.from_pydict(lowercase__)
																			yield file_idx, self._cast_table(lowercase__)

													# If the file has one json object per line
													else:
																			with open(lowercase__							,    '''rb''') as f:
																									__UpperCAmelCase :       Any  						=  0
																									# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																									# Set a default minimum value of 16kB if the chunk size is really small
																									__UpperCAmelCase :       Dict  						=  max(self.config.chunksize // 3_2							,    1_6 << 1_0)
																									__UpperCAmelCase :       Any  						=  (
																									    self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
																									)
																									while True:
																															__UpperCAmelCase :       Optional[Any]  						=  f.read(self.config.chunksize)
																															if not batch:
																																					break
																															# Finish current line
																															try:
																																					batch += f.readline()
																															except (AttributeError, io.UnsupportedOperation):
																																					batch += readline(lowercase__)
																															# PyArrow only accepts utf-8 encoded bytes
																															if self.config.encoding != "utf-8":
																																					__UpperCAmelCase :       Optional[Any]  						=  batch.decode(self.config.encoding							,    errors=lowercase__).encode('''utf-8''')
																															try:
																																					while True:
																																											try:
																																																	__UpperCAmelCase :       str  						=  paj.read_json(
																																																	    io.BytesIO(lowercase__)							,    read_options=paj.ReadOptions(block_size=lowercase__))
																																																	break
																																											except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																	if (
																																																	    isinstance(lowercase__							,    pa.ArrowInvalid)
																																																	    and "straddling" not in str(lowercase__)
																																																	    or block_size > len(lowercase__)
																																																	):
																																																							raise
																																																	else:
																																																							# Increase the block size in case it was too small.
																																																							# The block size will be reset for the next file.
																																																							logger.debug(
																																																							    F"Batch of {len(lowercase__)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.")
																																																							block_size *= 2
																															except pa.ArrowInvalid as e:
																																					try:
																																											with open(
																																											    lowercase__							,    encoding=self.config.encoding							,    errors=self.config.encoding_errors) as f:
																																																	__UpperCAmelCase :       int  						=  json.load(lowercase__)
																																					except json.JSONDecodeError:
																																											logger.error(F"Failed to read file '{file}' with error {type(lowercase__)}: {e}")
																																											raise e
																																					# If possible, parse the file as a list of json objects and exit the loop
																																					if isinstance(lowercase__							,    lowercase__):  # list is the only sequence type supported in JSON
																																											try:
																																																	__UpperCAmelCase :       Optional[Any]  						=  set().union(*[row.keys() for row in dataset])
																																																	__UpperCAmelCase :       Union[str, Any]  						=  {col: [row.get(lowercase__) for row in dataset] for col in keys}
																																																	__UpperCAmelCase :       Union[str, Any]  						=  pa.Table.from_pydict(lowercase__)
																																											except (pa.ArrowInvalid, AttributeError) as e:
																																																	logger.error(F"Failed to read file '{file}' with error {type(lowercase__)}: {e}")
																																																	raise ValueError(F"Not able to read records in the JSON file at {file}.") from None
																																											yield file_idx, self._cast_table(lowercase__)
																																											break
																																					else:
																																											logger.error(F"Failed to read file '{file}' with error {type(lowercase__)}: {e}")
																																											raise ValueError(
																																											    F"Not able to read records in the JSON file at {file}. "
																																											    F"You should probably indicate the field of the JSON file containing your records. "
																																											    F"This JSON file contain the following fields: {str(list(dataset.keys()))}. "
																																											    F"Select the correct one and provide it as `field='XXX'` to the dataset loading method. ") from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																															yield (file_idx, batch_idx), self._cast_table(lowercase__)
																															batch_idx += 1





import unittest

from datasets import load_dataset

from transformers.pipelines import pipeline
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_torch, slow




@is_pipeline_test
@require_torch
class        lowerCamelCase      (     unittest.TestCase   ):


	@require_torch
	def    A(						self):
							__UpperCAmelCase :       str  						=  pipeline(
							    task='''zero-shot-audio-classification'''							,    model='''hf-internal-testing/tiny-clap-htsat-unfused''')
							__UpperCAmelCase :       Optional[int]  						=  load_dataset('''ashraq/esc50''')
							__UpperCAmelCase :       Dict  						=  dataset['''train''']['''audio'''][-1]['''array''']
							__UpperCAmelCase :       Union[str, Any]  						=  audio_classifier(lowercase__							,    candidate_labels=['''Sound of a dog''', '''Sound of vaccum cleaner'''])
							self.assertEqual(
							    nested_simplify(lowercase__)							,    [{'''score''': 0.5_0_1, '''label''': '''Sound of a dog'''}, {'''score''': 0.4_9_9, '''label''': '''Sound of vaccum cleaner'''}]							,    )


	@unittest.skip('''No models are available in TF''')
	def    A(						self):
							pass


	@slow
	@require_torch
	def    A(						self):
							__UpperCAmelCase :       int  						=  pipeline(
							    task='''zero-shot-audio-classification'''							,    model='''laion/clap-htsat-unfused'''							,    )
							# This is an audio of a dog
							__UpperCAmelCase :       Optional[Any]  						=  load_dataset('''ashraq/esc50''')
							__UpperCAmelCase :       Union[str, Any]  						=  dataset['''train''']['''audio'''][-1]['''array''']
							__UpperCAmelCase :       Union[str, Any]  						=  audio_classifier(lowercase__							,    candidate_labels=['''Sound of a dog''', '''Sound of vaccum cleaner'''])

							self.assertEqual(
							    nested_simplify(lowercase__)							,    [
							        {'''score''': 0.9_9_9, '''label''': '''Sound of a dog'''},
							        {'''score''': 0.0_0_1, '''label''': '''Sound of vaccum cleaner'''},
							    ]							,    )

							__UpperCAmelCase :       Optional[Any]  						=  audio_classifier([audio] * 5							,    candidate_labels=['''Sound of a dog''', '''Sound of vaccum cleaner'''])
							self.assertEqual(
							    nested_simplify(lowercase__)							,    [
							        [
							            {'''score''': 0.9_9_9, '''label''': '''Sound of a dog'''},
							            {'''score''': 0.0_0_1, '''label''': '''Sound of vaccum cleaner'''},
							        ],
							    ]
							    * 5							,    )
							__UpperCAmelCase :       Optional[Any]  						=  audio_classifier(
							    [audio] * 5							,    candidate_labels=['''Sound of a dog''', '''Sound of vaccum cleaner''']							,    batch_size=5)
							self.assertEqual(
							    nested_simplify(lowercase__)							,    [
							        [
							            {'''score''': 0.9_9_9, '''label''': '''Sound of a dog'''},
							            {'''score''': 0.0_0_1, '''label''': '''Sound of vaccum cleaner'''},
							        ],
							    ]
							    * 5							,    )


	@unittest.skip('''No models are available in TF''')
	def    A(						self):
							pass

