






'''simple docstring'''







import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_A			:    Union[str, Any]					  =datasets.utils.logging.get_logger(__name__)

@dataclass
class _lowercase			(       datasets.BuilderConfig	):
							a          = None
							a          = """utf-8"""
							a          = None
							a          = None
							a          = True  # deprecated
							a          = None  # deprecated
							a          = 10 << 20  # 10MB
							a          = None



class _lowercase			(       datasets.ArrowBasedBuilder	):
							a          = JsonConfig




							def 	lowerCamelCase_       (							self:       str  ):
														if self.config.block_size is not None:
																					logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""  )
																					lowerCamelCase__      :			str 						= self.config.block_size
														if self.config.use_threads is not True:
																					logger.warning(
																					    """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."""  )
														if self.config.newlines_in_values is not None:
																					raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""  )
														return datasets.DatasetInfo(features=self.config.features  )




							def 	lowerCamelCase_       (							self:       List[str]    ,							UpperCamelCase__:       str  ):
														if not self.config.data_files:
																					raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''  )
														lowerCamelCase__      :			Tuple 						= dl_manager.download_and_extract(self.config.data_files  )
														if isinstance(UpperCamelCase__    ,							(str, list, tuple)  ):
																					lowerCamelCase__      :			Optional[Any] 						= data_files
																					if isinstance(UpperCamelCase__    ,							UpperCamelCase__  ):
																												lowerCamelCase__      :			List[str] 						= [files]
																					lowerCamelCase__      :			str 						= [dl_manager.iter_files(UpperCamelCase__  ) for file in files]
																					return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,							gen_kwargs={"""files""": files}  )]
														lowerCamelCase__      :			Union[str, Any] 						= []
														for split_name, files in data_files.items():
																					if isinstance(UpperCamelCase__    ,							UpperCamelCase__  ):
																												lowerCamelCase__      :			List[Any] 						= [files]
																					lowerCamelCase__      :			Dict 						= [dl_manager.iter_files(UpperCamelCase__  ) for file in files]
																					splits.append(datasets.SplitGenerator(name=UpperCamelCase__    ,							gen_kwargs={"""files""": files}  )  )
														return splits




							def 	lowerCamelCase_       (							self:       str    ,							UpperCamelCase__:       pa.Table  ):
														if self.config.features is not None:
																					# adding missing columns
																					for column_name in set(self.config.features  ) - set(pa_table.column_names  ):
																												lowerCamelCase__      :			Tuple 						= self.config.features.arrow_schema.field(UpperCamelCase__  ).type
																												lowerCamelCase__      :			Any 						= pa_table.append_column(UpperCamelCase__    ,							pa.array([None] * len(UpperCamelCase__  )    ,							type=UpperCamelCase__  )  )
																					# more expensive cast to support nested structures with keys in a different order
																					# allows str <-> int/float or str to Audio for example
																					lowerCamelCase__      :			Optional[Any] 						= table_cast(UpperCamelCase__    ,							self.config.features.arrow_schema  )
														return pa_table




							def 	lowerCamelCase_       (							self:       Union[str, Any]    ,							UpperCamelCase__:       Optional[int]  ):
														for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase__  )  ):
																					# If the file is one json object and if we need to look at the list of items in one specific field
																					if self.config.field is not None:
																												with open(UpperCamelCase__    ,							encoding=self.config.encoding    ,							errors=self.config.encoding_errors  ) as f:
																																			lowerCamelCase__      :			Union[str, Any] 						= json.load(UpperCamelCase__  )

																												# We keep only the field we are interested in
																												lowerCamelCase__      :			Optional[int] 						= dataset[self.config.field]

																												# We accept two format: a list of dicts or a dict of lists
																												if isinstance(UpperCamelCase__    ,							(list, tuple)  ):
																																			lowerCamelCase__      :			Any 						= set().union(*[row.keys() for row in dataset]  )
																																			lowerCamelCase__      :			Tuple 						= {col: [row.get(UpperCamelCase__  ) for row in dataset] for col in keys}
																												else:
																																			lowerCamelCase__      :			Optional[Any] 						= dataset
																												lowerCamelCase__      :			Dict 						= pa.Table.from_pydict(UpperCamelCase__  )
																												yield file_idx, self._cast_table(UpperCamelCase__  )

																					# If the file has one json object per line
																					else:
																												with open(UpperCamelCase__    ,							"""rb"""  ) as f:
																																			lowerCamelCase__      :			Dict 						= 0
																																			# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																																			# Set a default minimum value of 16kB if the chunk size is really small
																																			lowerCamelCase__      :			Tuple 						= max(self.config.chunksize // 32    ,							16 << 10  )
																																			lowerCamelCase__      :			List[str] 						= (
																																			    self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
																																			)
																																			while True:
																																										lowerCamelCase__      :			List[str] 						= f.read(self.config.chunksize  )
																																										if not batch:
																																																	break
																																										# Finish current line
																																										try:
																																																	batch += f.readline()
																																										except (AttributeError, io.UnsupportedOperation):
																																																	batch += readline(UpperCamelCase__  )
																																										# PyArrow only accepts utf-8 encoded bytes
																																										if self.config.encoding != "utf-8":
																																																	lowerCamelCase__      :			Union[str, Any] 						= batch.decode(self.config.encoding    ,							errors=UpperCamelCase__  ).encode("""utf-8"""  )
																																										try:
																																																	while True:
																																																								try:
																																																															lowerCamelCase__      :			List[str] 						= paj.read_json(
																																																															    io.BytesIO(UpperCamelCase__  )    ,							read_options=paj.ReadOptions(block_size=UpperCamelCase__  )  )
																																																															break
																																																								except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																															if (
																																																															    isinstance(UpperCamelCase__    ,							pa.ArrowInvalid  )
																																																															    and "straddling" not in str(UpperCamelCase__  )
																																																															    or block_size > len(UpperCamelCase__  )
																																																															):
																																																																						raise
																																																															else:
																																																																						# Increase the block size in case it was too small.
																																																																						# The block size will be reset for the next file.
																																																																						logger.debug(
																																																																						    F'''Batch of {len(UpperCamelCase__  )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''  )
																																																																						block_size *= 2
																																										except pa.ArrowInvalid as e:
																																																	try:
																																																								with open(
																																																								    UpperCamelCase__    ,							encoding=self.config.encoding    ,							errors=self.config.encoding_errors  ) as f:
																																																															lowerCamelCase__      :			Tuple 						= json.load(UpperCamelCase__  )
																																																	except json.JSONDecodeError:
																																																								logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase__  )}: {e}'''  )
																																																								raise e
																																																	# If possible, parse the file as a list of json objects and exit the loop
																																																	if isinstance(UpperCamelCase__    ,							UpperCamelCase__  ):  # list is the only sequence type supported in JSON
																																																								try:
																																																															lowerCamelCase__      :			Tuple 						= set().union(*[row.keys() for row in dataset]  )
																																																															lowerCamelCase__      :			Any 						= {col: [row.get(UpperCamelCase__  ) for row in dataset] for col in keys}
																																																															lowerCamelCase__      :			List[Any] 						= pa.Table.from_pydict(UpperCamelCase__  )
																																																								except (pa.ArrowInvalid, AttributeError) as e:
																																																															logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase__  )}: {e}'''  )
																																																															raise ValueError(F'''Not able to read records in the JSON file at {file}.'''  ) from None
																																																								yield file_idx, self._cast_table(UpperCamelCase__  )
																																																								break
																																																	else:
																																																								logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase__  )}: {e}'''  )
																																																								raise ValueError(
																																																								    F'''Not able to read records in the JSON file at {file}. '''
																																																								    F'''You should probably indicate the field of the JSON file containing your records. '''
																																																								    F'''This JSON file contain the following fields: {str(list(dataset.keys()  )  )}. '''
																																																								    F'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''  ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																										yield (file_idx, batch_idx), self._cast_table(UpperCamelCase__  )
																																										batch_idx += 1









'''simple docstring'''







import argparse
import os
import re

import packaging.version


_A			:    List[str]					  ='''examples/'''
_A			:    Any					  ={
    '''examples''': (re.compile(r'''^check_min_version\("[^"]+"\)\s*$''', re.MULTILINE), '''check_min_version("VERSION")\n'''),
    '''init''': (re.compile(r'''^__version__\s+=\s+"([^"]+)"\s*$''', re.MULTILINE), '''__version__ = "VERSION"\n'''),
    '''setup''': (re.compile(r'''^(\s*)version\s*=\s*"[^"]+",''', re.MULTILINE), r'''\1version="VERSION",'''),
    '''doc''': (re.compile(r'''^(\s*)release\s*=\s*"[^"]+"$''', re.MULTILINE), '''release = "VERSION"\n'''),
}
_A			:    int					  ={
    '''init''': '''src/transformers/__init__.py''',
    '''setup''': '''setup.py''',
}
_A			:    int					  ='''README.md'''





def 				SCREAMING_SNAKE_CASE_ (UpperCamelCase       ,    UpperCamelCase       ,    UpperCamelCase   )   -> str:
							with open(UpperCamelCase       ,    """r"""       ,    encoding="""utf-8"""       ,    newline="""\n"""   ) as f:
														lowerCamelCase__      :			List[str] 						= f.read()
							lowerCamelCase__						,	lowerCamelCase__      :			Optional[int] 						= REPLACE_PATTERNS[pattern]
							lowerCamelCase__      :			Dict 						= replace.replace("""VERSION"""       ,    UpperCamelCase   )
							lowerCamelCase__      :			str 						= re_pattern.sub(UpperCamelCase       ,    UpperCamelCase   )
							with open(UpperCamelCase       ,    """w"""       ,    encoding="""utf-8"""       ,    newline="""\n"""   ) as f:
														f.write(UpperCamelCase   )





def 				SCREAMING_SNAKE_CASE_ (UpperCamelCase   )   -> str:
							for folder, directories, fnames in os.walk(UpperCamelCase   ):
														# Removing some of the folders with non-actively maintained examples from the walk
														if "research_projects" in directories:
																					directories.remove("""research_projects"""   )
														if "legacy" in directories:
																					directories.remove("""legacy"""   )
														for fname in fnames:
																					if fname.endswith(""".py"""   ):
																												update_version_in_file(os.path.join(UpperCamelCase       ,    UpperCamelCase   )       ,    UpperCamelCase       ,    pattern="""examples"""   )





def 				SCREAMING_SNAKE_CASE_ (UpperCamelCase       ,    UpperCamelCase=False   )   -> List[Any]:
							for pattern, fname in REPLACE_FILES.items():
														update_version_in_file(UpperCamelCase       ,    UpperCamelCase       ,    UpperCamelCase   )
							if not patch:
														update_version_in_examples(UpperCamelCase   )





def 				SCREAMING_SNAKE_CASE_ ()   -> Optional[Any]:
							lowerCamelCase__      :			Dict 						= """🤗 Transformers currently provides the following architectures"""
							lowerCamelCase__      :			Dict 						= """1. Want to contribute a new model?"""
							with open(UpperCamelCase       ,    """r"""       ,    encoding="""utf-8"""       ,    newline="""\n"""   ) as f:
														lowerCamelCase__      :			int 						= f.readlines()

							# Find the start of the list.
							lowerCamelCase__      :			Optional[int] 						= 0
							while not lines[start_index].startswith(_start_prompt   ):
														start_index += 1
							start_index += 1

							lowerCamelCase__      :			Optional[Any] 						= start_index
							# Update the lines in the model list.
							while not lines[index].startswith(_end_prompt   ):
														if lines[index].startswith("""1."""   ):
																					lowerCamelCase__      :			List[Any] 						= lines[index].replace(
																					    """https://huggingface.co/docs/transformers/main/model_doc"""       ,    """https://huggingface.co/docs/transformers/model_doc"""       ,    )
														index += 1

							with open(UpperCamelCase       ,    """w"""       ,    encoding="""utf-8"""       ,    newline="""\n"""   ) as f:
														f.writelines(UpperCamelCase   )





def 				SCREAMING_SNAKE_CASE_ ()   -> Optional[Any]:
							with open(REPLACE_FILES["""init"""]       ,    """r"""   ) as f:
														lowerCamelCase__      :			int 						= f.read()
							lowerCamelCase__      :			Optional[Any] 						= REPLACE_PATTERNS["""init"""][0].search(UpperCamelCase   ).groups()[0]
							return packaging.version.parse(UpperCamelCase   )





def 				SCREAMING_SNAKE_CASE_ (UpperCamelCase=False   )   -> List[Any]:
							lowerCamelCase__      :			Union[str, Any] 						= get_version()
							if patch and default_version.is_devrelease:
														raise ValueError("""Can't create a patch version from the dev branch, checkout a released version!"""   )
							if default_version.is_devrelease:
														lowerCamelCase__      :			List[str] 						= default_version.base_version
							elif patch:
														lowerCamelCase__      :			Any 						= f'''{default_version.major}.{default_version.minor}.{default_version.micro + 1}'''
							else:
														lowerCamelCase__      :			List[Any] 						= f'''{default_version.major}.{default_version.minor + 1}.0'''

							# Now let's ask nicely if that's the right one.
							lowerCamelCase__      :			Any 						= input(f'''Which version are you releasing? [{default_version}]'''   )
							if len(UpperCamelCase   ) == 0:
														lowerCamelCase__      :			Optional[int] 						= default_version

							print(f'''Updating version to {version}.'''   )
							global_version_update(UpperCamelCase       ,    patch=UpperCamelCase   )
							if not patch:
														print("""Cleaning main README, don't forget to run `make fix-copies`."""   )
														clean_main_ref_in_model_list()





def 				SCREAMING_SNAKE_CASE_ ()   -> List[str]:
							lowerCamelCase__      :			Optional[int] 						= get_version()
							lowerCamelCase__      :			Any 						= f'''{current_version.major}.{current_version.minor + 1}.0.dev0'''
							lowerCamelCase__      :			Any 						= current_version.base_version

							# Check with the user we got that right.
							lowerCamelCase__      :			List[Any] 						= input(f'''Which version are we developing now? [{dev_version}]'''   )
							if len(UpperCamelCase   ) == 0:
														lowerCamelCase__      :			Dict 						= dev_version

							print(f'''Updating version to {version}.'''   )
							global_version_update(UpperCamelCase   )
							print("""Cleaning main README, don't forget to run `make fix-copies`."""   )
							clean_main_ref_in_model_list()


if __name__ == "__main__":
			_A			:    List[Any]					  =argparse.ArgumentParser()
			parser.add_argument('''--post_release''', action='''store_true''', help='''Whether this is pre or post release.''')
			parser.add_argument('''--patch''', action='''store_true''', help='''Whether or not this is a patch release.''')
			_A			:    List[str]					  =parser.parse_args()
			if not args.post_release:
						pre_release_work(patch=args.patch)
			elif args.patch:
						print('''Nothing to do after a patch :-)''')
			else:
						post_release_work()

