

import inspect
import unittest

from transformers import SegformerConfig, is_torch_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_MAPPING,
							    SegformerForImageClassification,
							    SegformerForSemanticSegmentation,
							    SegformerModel,
							)
							from transformers.models.segformer.modeling_segformer import SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
							from PIL import Image

							from transformers import SegformerImageProcessor



class       __snake_case  (    _UpperCamelCase       ):

		def 						lowerCAmelCase__     (					self):
						SCREAMING_SNAKE_CASE_									=						self.config_class(**self.inputs_dict)
						self.parent.assertTrue(hasattr(_A						,		'hidden_sizes'))
						self.parent.assertTrue(hasattr(_A						,		'num_attention_heads'))
						self.parent.assertTrue(hasattr(_A						,		'num_encoder_blocks'))






class       __snake_case  :

		def __init__(					self						,		_A						,		_A=13						,		_A=64						,		_A=3						,		_A=4						,		_A=[2, 2, 2, 2]						,		_A=[8, 4, 2, 1]						,		_A=[16, 32, 64, 128]						,		_A=[1, 4, 8, 16]						,		_A=[1, 2, 4, 8]						,		_A=True						,		_A=True						,		_A="gelu"						,		_A=0.1						,		_A=0.1						,		_A=0.0_2						,		_A=3						,		_A=None						,		):
						SCREAMING_SNAKE_CASE_									=						parent
						SCREAMING_SNAKE_CASE_									=						batch_size
						SCREAMING_SNAKE_CASE_									=						image_size
						SCREAMING_SNAKE_CASE_									=						num_channels
						SCREAMING_SNAKE_CASE_									=						num_encoder_blocks
						SCREAMING_SNAKE_CASE_									=						sr_ratios
						SCREAMING_SNAKE_CASE_									=						depths
						SCREAMING_SNAKE_CASE_									=						hidden_sizes
						SCREAMING_SNAKE_CASE_									=						downsampling_rates
						SCREAMING_SNAKE_CASE_									=						num_attention_heads
						SCREAMING_SNAKE_CASE_									=						is_training
						SCREAMING_SNAKE_CASE_									=						use_labels
						SCREAMING_SNAKE_CASE_									=						hidden_act
						SCREAMING_SNAKE_CASE_									=						hidden_dropout_prob
						SCREAMING_SNAKE_CASE_									=						attention_probs_dropout_prob
						SCREAMING_SNAKE_CASE_									=						initializer_range
						SCREAMING_SNAKE_CASE_									=						num_labels
						SCREAMING_SNAKE_CASE_									=						scope

		def 						lowerCAmelCase__     (					self):
						SCREAMING_SNAKE_CASE_									=						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

						SCREAMING_SNAKE_CASE_									=						None
						if self.use_labels:
										SCREAMING_SNAKE_CASE_									=						ids_tensor([self.batch_size, self.image_size, self.image_size]						,		self.num_labels)

						SCREAMING_SNAKE_CASE_									=						self.get_config()
						return config, pixel_values, labels

		def 						lowerCAmelCase__     (					self):
						return SegformerConfig(
						    image_size=self.image_size						,		num_channels=self.num_channels						,		num_encoder_blocks=self.num_encoder_blocks						,		depths=self.depths						,		hidden_sizes=self.hidden_sizes						,		num_attention_heads=self.num_attention_heads						,		hidden_act=self.hidden_act						,		hidden_dropout_prob=self.hidden_dropout_prob						,		attention_probs_dropout_prob=self.attention_probs_dropout_prob						,		initializer_range=self.initializer_range						,		)

		def 						lowerCAmelCase__     (					self						,		_A						,		_A						,		_A):
						SCREAMING_SNAKE_CASE_									=						SegformerModel(config=_A)
						model.to(_A)
						model.eval()
						SCREAMING_SNAKE_CASE_									=						model(_A)
						SCREAMING_SNAKE_CASE_									=						self.image_size // (self.downsampling_rates[-1] * 2)
						self.parent.assertEqual(
						    result.last_hidden_state.shape						,		(self.batch_size, self.hidden_sizes[-1], expected_height, expected_width))

		def 						lowerCAmelCase__     (					self						,		_A						,		_A						,		_A):
						SCREAMING_SNAKE_CASE_									=						self.num_labels
						SCREAMING_SNAKE_CASE_									=						SegformerForSemanticSegmentation(_A)
						model.to(_A)
						model.eval()
						SCREAMING_SNAKE_CASE_									=						model(_A)
						self.parent.assertEqual(
						    result.logits.shape						,		(self.batch_size, self.num_labels, self.image_size // 4, self.image_size // 4))
						SCREAMING_SNAKE_CASE_									=						model(_A						,		labels=_A)
						self.parent.assertEqual(
						    result.logits.shape						,		(self.batch_size, self.num_labels, self.image_size // 4, self.image_size // 4))
						self.parent.assertGreater(result.loss						,		0.0)

		def 						lowerCAmelCase__     (					self						,		_A						,		_A						,		_A):
						SCREAMING_SNAKE_CASE_									=						1
						SCREAMING_SNAKE_CASE_									=						SegformerForSemanticSegmentation(config=_A)
						model.to(_A)
						model.eval()
						SCREAMING_SNAKE_CASE_									=						torch.randint(0						,		1						,		(self.batch_size, self.image_size, self.image_size)).to(_A)
						SCREAMING_SNAKE_CASE_									=						model(_A						,		labels=_A)
						self.parent.assertGreater(result.loss						,		0.0)


		def 						lowerCAmelCase__     (					self):
						SCREAMING_SNAKE_CASE_									=						self.prepare_config_and_inputs()
						SCREAMING_SNAKE_CASE_									=						config_and_inputs
						SCREAMING_SNAKE_CASE_									=						{"pixel_values": pixel_values}
						return config, inputs_dict






@require_torch
class       __snake_case  (    _UpperCamelCase     , _UpperCamelCase     , unittest.TestCase       ):
		__lowerCAmelCase     :	str						 =  (
		    (
		        SegformerModel,
		        SegformerForSemanticSegmentation,
		        SegformerForImageClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		__lowerCAmelCase     :	Optional[int]						 =  (
		    {
		        'feature-extraction': SegformerModel,
		        'image-classification': SegformerForImageClassification,
		        'image-segmentation': SegformerForSemanticSegmentation,
		    }
		    if is_torch_available()
		    else {}
		)

		__lowerCAmelCase     :	Optional[int]						 =  True
		__lowerCAmelCase     :	List[Any]						 =  False
		__lowerCAmelCase     :	Optional[int]						 =  False
		__lowerCAmelCase     :	List[Any]						 =  False

		def 						lowerCAmelCase__     (					self):
						SCREAMING_SNAKE_CASE_									=						SegformerModelTester(self)
						SCREAMING_SNAKE_CASE_									=						SegformerConfigTester(self						,		config_class=_A)

		def 						lowerCAmelCase__     (					self):
						self.config_tester.run_common_tests()

		def 						lowerCAmelCase__     (					self):
						SCREAMING_SNAKE_CASE_									=						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*_A)

		def 						lowerCAmelCase__     (					self):
						SCREAMING_SNAKE_CASE_									=						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_binary_image_segmentation(*_A)

		def 						lowerCAmelCase__     (					self):
						SCREAMING_SNAKE_CASE_									=						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_image_segmentation(*_A)

		@unittest.skip('SegFormer does not use inputs_embeds')
		def 						lowerCAmelCase__     (					self):
						pass

		@unittest.skip('SegFormer does not have get_input_embeddings method and get_output_embeddings methods')
		def 						lowerCAmelCase__     (					self):
						pass

		def 						lowerCAmelCase__     (					self):
						SCREAMING_SNAKE_CASE_									=						self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
										SCREAMING_SNAKE_CASE_									=						model_class(_A)
										SCREAMING_SNAKE_CASE_									=						inspect.signature(model.forward)
										# signature.parameters is an OrderedDict => so arg_names order is deterministic
										SCREAMING_SNAKE_CASE_									=						[*signature.parameters.keys()]

										SCREAMING_SNAKE_CASE_									=						["pixel_values"]
										self.assertListEqual(arg_names[:1]						,		_A)

		def 						lowerCAmelCase__     (					self):
						SCREAMING_SNAKE_CASE_									=						self.model_tester.prepare_config_and_inputs_for_common()
						SCREAMING_SNAKE_CASE_									=						True

						for model_class in self.all_model_classes:
										SCREAMING_SNAKE_CASE_									=						True
										SCREAMING_SNAKE_CASE_									=						False
										SCREAMING_SNAKE_CASE_									=						True
										SCREAMING_SNAKE_CASE_									=						model_class(_A)
										model.to(_A)
										model.eval()
										with torch.no_grad():
														SCREAMING_SNAKE_CASE_									=						model(**self._prepare_for_class(_A						,		_A))
										SCREAMING_SNAKE_CASE_									=						outputs.attentions

										SCREAMING_SNAKE_CASE_									=						sum(self.model_tester.depths)
										self.assertEqual(len(_A)						,		_A)

										# check that output_attentions also work using config
										del inputs_dict["output_attentions"]
										SCREAMING_SNAKE_CASE_									=						True
										SCREAMING_SNAKE_CASE_									=						model_class(_A)
										model.to(_A)
										model.eval()
										with torch.no_grad():
														SCREAMING_SNAKE_CASE_									=						model(**self._prepare_for_class(_A						,		_A))
										SCREAMING_SNAKE_CASE_									=						outputs.attentions

										self.assertEqual(len(_A)						,		_A)

										# verify the first attentions (first block, first layer)
										SCREAMING_SNAKE_CASE_									=						(self.model_tester.image_size // 4) ** 2
										SCREAMING_SNAKE_CASE_									=						(self.model_tester.image_size // (4 * self.model_tester.sr_ratios[0])) ** 2
										self.assertListEqual(
										    list(attentions[0].shape[-3:])						,		[self.model_tester.num_attention_heads[0], expected_seq_len, expected_reduced_seq_len]						,		)

										# verify the last attentions (last block, last layer)
										SCREAMING_SNAKE_CASE_									=						(self.model_tester.image_size // 32) ** 2
										SCREAMING_SNAKE_CASE_									=						(self.model_tester.image_size // (32 * self.model_tester.sr_ratios[-1])) ** 2
										self.assertListEqual(
										    list(attentions[-1].shape[-3:])						,		[self.model_tester.num_attention_heads[-1], expected_seq_len, expected_reduced_seq_len]						,		)
										SCREAMING_SNAKE_CASE_									=						len(_A)

										# Check attention is always last and order is fine
										SCREAMING_SNAKE_CASE_									=						True
										SCREAMING_SNAKE_CASE_									=						True
										SCREAMING_SNAKE_CASE_									=						model_class(_A)
										model.to(_A)
										model.eval()
										with torch.no_grad():
														SCREAMING_SNAKE_CASE_									=						model(**self._prepare_for_class(_A						,		_A))

										self.assertEqual(out_len + 1						,		len(_A))

										SCREAMING_SNAKE_CASE_									=						outputs.attentions

										self.assertEqual(len(_A)						,		_A)
										# verify the first attentions (first block, first layer)
										SCREAMING_SNAKE_CASE_									=						(self.model_tester.image_size // 4) ** 2
										SCREAMING_SNAKE_CASE_									=						(self.model_tester.image_size // (4 * self.model_tester.sr_ratios[0])) ** 2
										self.assertListEqual(
										    list(self_attentions[0].shape[-3:])						,		[self.model_tester.num_attention_heads[0], expected_seq_len, expected_reduced_seq_len]						,		)

		def 						lowerCAmelCase__     (					self):
						def check_hidden_states_output(_A						,		_A						,		_A):
										SCREAMING_SNAKE_CASE_									=						model_class(_A)
										model.to(_A)
										model.eval()

										with torch.no_grad():
														SCREAMING_SNAKE_CASE_									=						model(**self._prepare_for_class(_A						,		_A))

										SCREAMING_SNAKE_CASE_									=						outputs.hidden_states

										SCREAMING_SNAKE_CASE_									=						self.model_tester.num_encoder_blocks
										self.assertEqual(len(_A)						,		_A)

										# verify the first hidden states (first block)
										self.assertListEqual(
										    list(hidden_states[0].shape[-3:])						,		[
										        self.model_tester.hidden_sizes[0],
										        self.model_tester.image_size // 4,
										        self.model_tester.image_size // 4,
										    ]						,		)

						SCREAMING_SNAKE_CASE_									=						self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
										SCREAMING_SNAKE_CASE_									=						True
										check_hidden_states_output(_A						,		_A						,		_A)

										# check that output_hidden_states also work using config
										del inputs_dict["output_hidden_states"]
										SCREAMING_SNAKE_CASE_									=						True

										check_hidden_states_output(_A						,		_A						,		_A)

		def 						lowerCAmelCase__     (					self):
						if not self.model_tester.is_training:
										return

						SCREAMING_SNAKE_CASE_									=						self.model_tester.prepare_config_and_inputs_for_common()
						SCREAMING_SNAKE_CASE_									=						True

						for model_class in self.all_model_classes:
										if model_class in get_values(_A):
														continue

										SCREAMING_SNAKE_CASE_									=						model_class(_A)
										model.to(_A)
										model.train()
										SCREAMING_SNAKE_CASE_									=						self._prepare_for_class(_A						,		_A						,		return_labels=_A)
										SCREAMING_SNAKE_CASE_									=						model(**_A).loss
										loss.backward()

		@unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.')
		def 						lowerCAmelCase__     (					self):
						pass


		@slow
		def 						lowerCAmelCase__     (					self):
						for model_name in SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										SCREAMING_SNAKE_CASE_									=						SegformerModel.from_pretrained(_A)
										self.assertIsNotNone(_A)






def 					_UpperCAmelCase       (  ):





				"""simple docstring"""


				SCREAMING_SNAKE_CASE_									=						Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'       )
				return image





@require_torch
class       __snake_case  (    unittest.TestCase       ):

		@slow
		def 						lowerCAmelCase__     (					self):
						# only resize + normalize
						SCREAMING_SNAKE_CASE_									=						SegformerImageProcessor(
						    image_scale=(512, 512)						,		keep_ratio=_A						,		align=_A						,		do_random_crop=_A)
						SCREAMING_SNAKE_CASE_									=						SegformerForSemanticSegmentation.from_pretrained('nvidia/segformer-b0-finetuned-ade-512-512').to(
						    _A)

						SCREAMING_SNAKE_CASE_									=						prepare_img()
						SCREAMING_SNAKE_CASE_									=						image_processor(images=_A						,		return_tensors='pt')
						SCREAMING_SNAKE_CASE_									=						encoded_inputs.pixel_values.to(_A)

						with torch.no_grad():
										SCREAMING_SNAKE_CASE_									=						model(_A)

						SCREAMING_SNAKE_CASE_									=						torch.Size((1, model.config.num_labels, 128, 128))
						self.assertEqual(outputs.logits.shape						,		_A)

						SCREAMING_SNAKE_CASE_									=						torch.tensor(
						    [
						        [[-4.6_3_1_0, -5.5_2_3_2, -6.2_3_5_6], [-5.1_9_2_1, -6.1_4_4_4, -6.5_9_9_6], [-5.4_4_2_4, -6.2_7_9_0, -6.7_5_7_4]],
						        [[-1_2.1_3_9_1, -1_3.3_1_2_2, -1_3.9_5_5_4], [-1_2.8_7_3_2, -1_3.9_3_5_2, -1_4.3_5_6_3], [-1_2.9_4_3_8, -1_3.8_2_2_6, -1_4.2_5_1_3]],
						        [[-1_2.5_1_3_4, -1_3.4_6_8_6, -1_4.4_9_1_5], [-1_2.8_6_6_9, -1_4.4_3_4_3, -1_4.7_7_5_8], [-1_3.2_5_2_3, -1_4.5_8_1_9, -1_5.0_6_9_4]],
						    ]).to(_A)
						self.assertTrue(torch.allclose(outputs.logits[0, :3, :3, :3]						,		_A						,		atol=1E-4))

		@slow
		def 						lowerCAmelCase__     (					self):
						# only resize + normalize
						SCREAMING_SNAKE_CASE_									=						SegformerImageProcessor(
						    image_scale=(512, 512)						,		keep_ratio=_A						,		align=_A						,		do_random_crop=_A)
						SCREAMING_SNAKE_CASE_									=						SegformerForSemanticSegmentation.from_pretrained(
						    'nvidia/segformer-b1-finetuned-cityscapes-1024-1024').to(_A)

						SCREAMING_SNAKE_CASE_									=						prepare_img()
						SCREAMING_SNAKE_CASE_									=						image_processor(images=_A						,		return_tensors='pt')
						SCREAMING_SNAKE_CASE_									=						encoded_inputs.pixel_values.to(_A)

						with torch.no_grad():
										SCREAMING_SNAKE_CASE_									=						model(_A)

						SCREAMING_SNAKE_CASE_									=						torch.Size((1, model.config.num_labels, 128, 128))
						self.assertEqual(outputs.logits.shape						,		_A)

						SCREAMING_SNAKE_CASE_									=						torch.tensor(
						    [
						        [[-1_3.5_7_4_8, -1_3.9_1_1_1, -1_2.6_5_0_0], [-1_4.3_5_0_0, -1_5.3_6_8_3, -1_4.2_3_2_8], [-1_4.7_5_3_2, -1_6.0_4_2_4, -1_5.6_0_8_7]],
						        [[-1_7.1_6_5_1, -1_5.8_7_2_5, -1_2.9_6_5_3], [-1_7.2_5_8_0, -1_7.3_7_1_8, -1_4.8_2_2_3], [-1_6.6_0_5_8, -1_6.8_7_8_3, -1_6.7_4_5_2]],
						        [[-3.6_4_5_6, -3.0_2_0_9, -1.4_2_0_3], [-3.0_7_9_7, -3.1_9_5_9, -2.0_0_0_0], [-1.8_7_5_7, -1.9_2_1_7, -1.6_9_9_7]],
						    ]).to(_A)
						self.assertTrue(torch.allclose(outputs.logits[0, :3, :3, :3]						,		_A						,		atol=1E-1))


		@slow
		def 						lowerCAmelCase__     (					self):
						# only resize + normalize
						SCREAMING_SNAKE_CASE_									=						SegformerImageProcessor(
						    image_scale=(512, 512)						,		keep_ratio=_A						,		align=_A						,		do_random_crop=_A)
						SCREAMING_SNAKE_CASE_									=						SegformerForSemanticSegmentation.from_pretrained('nvidia/segformer-b0-finetuned-ade-512-512').to(
						    _A)

						SCREAMING_SNAKE_CASE_									=						prepare_img()
						SCREAMING_SNAKE_CASE_									=						image_processor(images=_A						,		return_tensors='pt')
						SCREAMING_SNAKE_CASE_									=						encoded_inputs.pixel_values.to(_A)

						with torch.no_grad():
										SCREAMING_SNAKE_CASE_									=						model(_A)

						SCREAMING_SNAKE_CASE_									=						outputs.logits.detach().cpu()

						SCREAMING_SNAKE_CASE_									=						image_processor.post_process_semantic_segmentation(outputs=_A						,		target_sizes=[(500, 300)])
						SCREAMING_SNAKE_CASE_									=						torch.Size((500, 300))
						self.assertEqual(segmentation[0].shape						,		_A)

						SCREAMING_SNAKE_CASE_									=						image_processor.post_process_semantic_segmentation(outputs=_A)
						SCREAMING_SNAKE_CASE_									=						torch.Size((128, 128))
						self.assertEqual(segmentation[0].shape						,		_A)





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


UpperCamelCase__  :      Optional[int]			=    datasets.utils.logging.get_logger(__name__)



@dataclass
class       __snake_case  (    datasets.BuilderConfig       ):
  __lowerCAmelCase     :	Optional[datasets.Features]						 =  None
  __lowerCAmelCase     :	str						 =  "utf-8"
  __lowerCAmelCase     :	Optional[str]						 =  None
  __lowerCAmelCase     :	Optional[str]						 =  None
  __lowerCAmelCase     :	bool						 =  True  # deprecated
  __lowerCAmelCase     :	Optional[int]						 =  None  # deprecated
  __lowerCAmelCase     :	int						 =  10 << 20  # 10MB
  __lowerCAmelCase     :	Optional[bool]						 =  None






class       __snake_case  (    datasets.ArrowBasedBuilder       ):
  __lowerCAmelCase     :	int						 =  JsonConfig

  def 						lowerCAmelCase__     (					self):
      if self.config.block_size is not None:
          logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead')
          SCREAMING_SNAKE_CASE_									=						self.config.block_size
      if self.config.use_threads is not True:
          logger.warning(
              'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.')
      if self.config.newlines_in_values is not None:
          raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported')
      return datasets.DatasetInfo(features=self.config.features)

  def 						lowerCAmelCase__     (					self						,		_A):
      if not self.config.data_files:
          raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}""")
      SCREAMING_SNAKE_CASE_									=						dl_manager.download_and_extract(self.config.data_files)
      if isinstance(_A						,		(str, list, tuple)):
          SCREAMING_SNAKE_CASE_									=						data_files
          if isinstance(_A						,		_A):
              SCREAMING_SNAKE_CASE_									=						[files]
          SCREAMING_SNAKE_CASE_									=						[dl_manager.iter_files(_A) for file in files]
          return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,		gen_kwargs={'files': files})]
      SCREAMING_SNAKE_CASE_									=						[]
      for split_name, files in data_files.items():
          if isinstance(_A						,		_A):
              SCREAMING_SNAKE_CASE_									=						[files]
          SCREAMING_SNAKE_CASE_									=						[dl_manager.iter_files(_A) for file in files]
          splits.append(datasets.SplitGenerator(name=_A						,		gen_kwargs={'files': files}))
      return splits

  def 						lowerCAmelCase__     (					self						,		_A):
      if self.config.features is not None:
          # adding missing columns
          for column_name in set(self.config.features) - set(pa_table.column_names):
              SCREAMING_SNAKE_CASE_									=						self.config.features.arrow_schema.field(_A).type
              SCREAMING_SNAKE_CASE_									=						pa_table.append_column(_A						,		pa.array([None] * len(_A)						,		type=_A))
          # more expensive cast to support nested structures with keys in a different order
          # allows str <-> int/float or str to Audio for example
          SCREAMING_SNAKE_CASE_									=						table_cast(_A						,		self.config.features.arrow_schema)
      return pa_table


  def 						lowerCAmelCase__     (					self						,		_A):
      for file_idx, file in enumerate(itertools.chain.from_iterable(_A)):
          # If the file is one json object and if we need to look at the list of items in one specific field
          if self.config.field is not None:
              with open(_A						,		encoding=self.config.encoding						,		errors=self.config.encoding_errors) as f:
                  SCREAMING_SNAKE_CASE_									=						json.load(_A)

              # We keep only the field we are interested in
              SCREAMING_SNAKE_CASE_									=						dataset[self.config.field]

              # We accept two format: a list of dicts or a dict of lists
              if isinstance(_A						,		(list, tuple)):
                  SCREAMING_SNAKE_CASE_									=						set().union(*[row.keys() for row in dataset])
                  SCREAMING_SNAKE_CASE_									=						{col: [row.get(_A) for row in dataset] for col in keys}
              else:
                  SCREAMING_SNAKE_CASE_									=						dataset
              SCREAMING_SNAKE_CASE_									=						pa.Table.from_pydict(_A)
              yield file_idx, self._cast_table(_A)

          # If the file has one json object per line
          else:
              with open(_A						,		'rb') as f:
                  SCREAMING_SNAKE_CASE_									=						0
                  # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                  # Set a default minimum value of 16kB if the chunk size is really small
                  SCREAMING_SNAKE_CASE_									=						max(self.config.chunksize // 32						,		16 << 10)
                  SCREAMING_SNAKE_CASE_									=						(
                      self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
                  )
                  while True:
                      SCREAMING_SNAKE_CASE_									=						f.read(self.config.chunksize)
                      if not batch:
                          break
                      # Finish current line
                      try:
                          batch += f.readline()
                      except (AttributeError, io.UnsupportedOperation):
                          batch += readline(_A)
                      # PyArrow only accepts utf-8 encoded bytes
                      if self.config.encoding != "utf-8":
                          SCREAMING_SNAKE_CASE_									=						batch.decode(self.config.encoding						,		errors=_A).encode('utf-8')
                      try:
                          while True:
                              try:
                                  SCREAMING_SNAKE_CASE_									=						paj.read_json(
                                      io.BytesIO(_A)						,		read_options=paj.ReadOptions(block_size=_A))
                                  break
                              except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                  if (
                                      isinstance(_A						,		pa.ArrowInvalid)
                                      and "straddling" not in str(_A)
                                      or block_size > len(_A)
                                  ):
                                      raise
                                  else:
                                      # Increase the block size in case it was too small.
                                      # The block size will be reset for the next file.
                                      logger.debug(
                                          f"""Batch of {len(_A)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.""")
                                      block_size *= 2
                      except pa.ArrowInvalid as e:
                          try:
                              with open(
                                  _A						,		encoding=self.config.encoding						,		errors=self.config.encoding_errors) as f:
                                  SCREAMING_SNAKE_CASE_									=						json.load(_A)
                          except json.JSONDecodeError:
                              logger.error(f"""Failed to read file '{file}' with error {type(_A)}: {e}""")
                              raise e
                          # If possible, parse the file as a list of json objects and exit the loop
                          if isinstance(_A						,		_A):  # list is the only sequence type supported in JSON
                              try:
                                  SCREAMING_SNAKE_CASE_									=						set().union(*[row.keys() for row in dataset])
                                  SCREAMING_SNAKE_CASE_									=						{col: [row.get(_A) for row in dataset] for col in keys}
                                  SCREAMING_SNAKE_CASE_									=						pa.Table.from_pydict(_A)
                              except (pa.ArrowInvalid, AttributeError) as e:
                                  logger.error(f"""Failed to read file '{file}' with error {type(_A)}: {e}""")
                                  raise ValueError(f"""Not able to read records in the JSON file at {file}.""") from None
                              yield file_idx, self._cast_table(_A)
                              break
                          else:
                              logger.error(f"""Failed to read file '{file}' with error {type(_A)}: {e}""")
                              raise ValueError(
                                  f"""Not able to read records in the JSON file at {file}. """
                                  f"""You should probably indicate the field of the JSON file containing your records. """
                                  f"""This JSON file contain the following fields: {str(list(dataset.keys()))}. """
                                  f"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                      yield (file_idx, batch_idx), self._cast_table(_A)
                      batch_idx += 1


