






'''simple docstring'''







import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_A			:    Union[str, Any]					  =datasets.utils.logging.get_logger(__name__)

@dataclass
class _lowercase			(       datasets.BuilderConfig	):
							a          = None
							a          = "utf-8"
							a          = None
							a          = None
							a          = True  # deprecated
							a          = None  # deprecated
							a          = 10 << 20  # 10MB
							a          = None



class _lowercase			(       datasets.ArrowBasedBuilder	):
							a          = JsonConfig




							def 	lowerCamelCase_       (							self:       str  ):
														if self.config.block_size is not None:
																					logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""  )
																					lowerCamelCase__      :			str 						= self.config.block_size
														if self.config.use_threads is not True:
																					logger.warning(
																					    """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."""  )
														if self.config.newlines_in_values is not None:
																					raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""  )
														return datasets.DatasetInfo(features=self.config.features  )




							def 	lowerCamelCase_       (							self:       List[str]    ,							UpperCamelCase__:       str  ):
														if not self.config.data_files:
																					raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''  )
														lowerCamelCase__      :			Tuple 						= dl_manager.download_and_extract(self.config.data_files  )
														if isinstance(UpperCamelCase__    ,							(str, list, tuple)  ):
																					lowerCamelCase__      :			Optional[Any] 						= data_files
																					if isinstance(UpperCamelCase__    ,							UpperCamelCase__  ):
																												lowerCamelCase__      :			List[str] 						= [files]
																					lowerCamelCase__      :			str 						= [dl_manager.iter_files(UpperCamelCase__  ) for file in files]
																					return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,							gen_kwargs={"""files""": files}  )]
														lowerCamelCase__      :			Union[str, Any] 						= []
														for split_name, files in data_files.items():
																					if isinstance(UpperCamelCase__    ,							UpperCamelCase__  ):
																												lowerCamelCase__      :			List[Any] 						= [files]
																					lowerCamelCase__      :			Dict 						= [dl_manager.iter_files(UpperCamelCase__  ) for file in files]
																					splits.append(datasets.SplitGenerator(name=UpperCamelCase__    ,							gen_kwargs={"""files""": files}  )  )
														return splits




							def 	lowerCamelCase_       (							self:       str    ,							UpperCamelCase__:       pa.Table  ):
														if self.config.features is not None:
																					# adding missing columns
																					for column_name in set(self.config.features  ) - set(pa_table.column_names  ):
																												lowerCamelCase__      :			Tuple 						= self.config.features.arrow_schema.field(UpperCamelCase__  ).type
																												lowerCamelCase__      :			Any 						= pa_table.append_column(UpperCamelCase__    ,							pa.array([None] * len(UpperCamelCase__  )    ,							type=UpperCamelCase__  )  )
																					# more expensive cast to support nested structures with keys in a different order
																					# allows str <-> int/float or str to Audio for example
																					lowerCamelCase__      :			Optional[Any] 						= table_cast(UpperCamelCase__    ,							self.config.features.arrow_schema  )
														return pa_table




							def 	lowerCamelCase_       (							self:       Union[str, Any]    ,							UpperCamelCase__:       Optional[int]  ):
														for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase__  )  ):
																					# If the file is one json object and if we need to look at the list of items in one specific field
																					if self.config.field is not None:
																												with open(UpperCamelCase__    ,							encoding=self.config.encoding    ,							errors=self.config.encoding_errors  ) as f:
																																			lowerCamelCase__      :			Union[str, Any] 						= json.load(UpperCamelCase__  )

																												# We keep only the field we are interested in
																												lowerCamelCase__      :			Optional[int] 						= dataset[self.config.field]

																												# We accept two format: a list of dicts or a dict of lists
																												if isinstance(UpperCamelCase__    ,							(list, tuple)  ):
																																			lowerCamelCase__      :			Any 						= set().union(*[row.keys() for row in dataset]  )
																																			lowerCamelCase__      :			Tuple 						= {col: [row.get(UpperCamelCase__  ) for row in dataset] for col in keys}
																												else:
																																			lowerCamelCase__      :			Optional[Any] 						= dataset
																												lowerCamelCase__      :			Dict 						= pa.Table.from_pydict(UpperCamelCase__  )
																												yield file_idx, self._cast_table(UpperCamelCase__  )

																					# If the file has one json object per line
																					else:
																												with open(UpperCamelCase__    ,							"""rb"""  ) as f:
																																			lowerCamelCase__      :			Dict 						= 0
																																			# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																																			# Set a default minimum value of 16kB if the chunk size is really small
																																			lowerCamelCase__      :			Tuple 						= max(self.config.chunksize // 32    ,							16 << 10  )
																																			lowerCamelCase__      :			List[str] 						= (
																																			    self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
																																			)
																																			while True:
																																										lowerCamelCase__      :			List[str] 						= f.read(self.config.chunksize  )
																																										if not batch:
																																																	break
																																										# Finish current line
																																										try:
																																																	batch += f.readline()
																																										except (AttributeError, io.UnsupportedOperation):
																																																	batch += readline(UpperCamelCase__  )
																																										# PyArrow only accepts utf-8 encoded bytes
																																										if self.config.encoding != "utf-8":
																																																	lowerCamelCase__      :			Union[str, Any] 						= batch.decode(self.config.encoding    ,							errors=UpperCamelCase__  ).encode("""utf-8"""  )
																																										try:
																																																	while True:
																																																								try:
																																																															lowerCamelCase__      :			List[str] 						= paj.read_json(
																																																															    io.BytesIO(UpperCamelCase__  )    ,							read_options=paj.ReadOptions(block_size=UpperCamelCase__  )  )
																																																															break
																																																								except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																															if (
																																																															    isinstance(UpperCamelCase__    ,							pa.ArrowInvalid  )
																																																															    and "straddling" not in str(UpperCamelCase__  )
																																																															    or block_size > len(UpperCamelCase__  )
																																																															):
																																																																						raise
																																																															else:
																																																																						# Increase the block size in case it was too small.
																																																																						# The block size will be reset for the next file.
																																																																						logger.debug(
																																																																						    F'''Batch of {len(UpperCamelCase__  )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''  )
																																																																						block_size *= 2
																																										except pa.ArrowInvalid as e:
																																																	try:
																																																								with open(
																																																								    UpperCamelCase__    ,							encoding=self.config.encoding    ,							errors=self.config.encoding_errors  ) as f:
																																																															lowerCamelCase__      :			Tuple 						= json.load(UpperCamelCase__  )
																																																	except json.JSONDecodeError:
																																																								logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase__  )}: {e}'''  )
																																																								raise e
																																																	# If possible, parse the file as a list of json objects and exit the loop
																																																	if isinstance(UpperCamelCase__    ,							UpperCamelCase__  ):  # list is the only sequence type supported in JSON
																																																								try:
																																																															lowerCamelCase__      :			Tuple 						= set().union(*[row.keys() for row in dataset]  )
																																																															lowerCamelCase__      :			Any 						= {col: [row.get(UpperCamelCase__  ) for row in dataset] for col in keys}
																																																															lowerCamelCase__      :			List[Any] 						= pa.Table.from_pydict(UpperCamelCase__  )
																																																								except (pa.ArrowInvalid, AttributeError) as e:
																																																															logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase__  )}: {e}'''  )
																																																															raise ValueError(F'''Not able to read records in the JSON file at {file}.'''  ) from None
																																																								yield file_idx, self._cast_table(UpperCamelCase__  )
																																																								break
																																																	else:
																																																								logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase__  )}: {e}'''  )
																																																								raise ValueError(
																																																								    F'''Not able to read records in the JSON file at {file}. '''
																																																								    F'''You should probably indicate the field of the JSON file containing your records. '''
																																																								    F'''This JSON file contain the following fields: {str(list(dataset.keys()  )  )}. '''
																																																								    F'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''  ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																										yield (file_idx, batch_idx), self._cast_table(UpperCamelCase__  )
																																										batch_idx += 1









'''simple docstring'''







from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A			:    Any					  =logging.get_logger(__name__)

_A			:    Dict					  ={
    '''microsoft/trocr-base-handwritten''': (
        '''https://huggingface.co/microsoft/trocr-base-handwritten/resolve/main/config.json'''
    ),
    # See all TrOCR models at https://huggingface.co/models?filter=trocr
}

class _lowercase			(       _lowercase	):
							a          = """trocr"""
							a          = ["""past_key_values"""]
							a          = {
							    """num_attention_heads""": """decoder_attention_heads""",
							    """hidden_size""": """d_model""",
							    """num_hidden_layers""": """decoder_layers""",
							}




							def __init__(							self:       Optional[Any]    ,							UpperCamelCase__:       int=50_265    ,							UpperCamelCase__:       int=1_024    ,							UpperCamelCase__:       Optional[Any]=12    ,							UpperCamelCase__:       Dict=16    ,							UpperCamelCase__:       int=4_096    ,							UpperCamelCase__:       Tuple="gelu"    ,							UpperCamelCase__:       int=512    ,							UpperCamelCase__:       Dict=0.1    ,							UpperCamelCase__:       Tuple=0.0    ,							UpperCamelCase__:       str=0.0    ,							UpperCamelCase__:       Any=2    ,							UpperCamelCase__:       Dict=0.02    ,							UpperCamelCase__:       Optional[Any]=0.0    ,							UpperCamelCase__:       str=True    ,							UpperCamelCase__:       Tuple=False    ,							UpperCamelCase__:       Union[str, Any]=True    ,							UpperCamelCase__:       Tuple=True    ,							UpperCamelCase__:       Dict=1    ,							UpperCamelCase__:       List[str]=0    ,							UpperCamelCase__:       Union[str, Any]=2    ,							**UpperCamelCase__:       str    ,							):
														lowerCamelCase__      :			Any 						= vocab_size
														lowerCamelCase__      :			Tuple 						= d_model
														lowerCamelCase__      :			Any 						= decoder_layers
														lowerCamelCase__      :			Dict 						= decoder_attention_heads
														lowerCamelCase__      :			str 						= decoder_ffn_dim
														lowerCamelCase__      :			Tuple 						= activation_function
														lowerCamelCase__      :			Dict 						= max_position_embeddings
														lowerCamelCase__      :			int 						= dropout
														lowerCamelCase__      :			int 						= attention_dropout
														lowerCamelCase__      :			List[Any] 						= activation_dropout
														lowerCamelCase__      :			Union[str, Any] 						= init_std
														lowerCamelCase__      :			Optional[int] 						= decoder_layerdrop
														lowerCamelCase__      :			Dict 						= use_cache
														lowerCamelCase__      :			Any 						= scale_embedding
														lowerCamelCase__      :			Optional[int] 						= use_learned_position_embeddings
														lowerCamelCase__      :			List[str] 						= layernorm_embedding

														super().__init__(
														    pad_token_id=UpperCamelCase__    ,							bos_token_id=UpperCamelCase__    ,							eos_token_id=UpperCamelCase__    ,							decoder_start_token_id=UpperCamelCase__    ,							**UpperCamelCase__    ,							)

