import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


_lowerCamelCase					:   Optional[Any]							=		datasets.utils.logging.get_logger(__name__)


@dataclass
class 						SCREAMING_SNAKE_CASE__    (     datasets.BuilderConfig     ):
							lowercase__						:       Optional[Any]								=							10_000
							lowercase__						:       int								=							None
							lowercase__						:       Optional[Any]								=							None



class 						SCREAMING_SNAKE_CASE__    (     datasets.ArrowBasedBuilder     ):
							lowercase__						:       Any								=							ParquetConfig




							def 	__snake_case(						self   :					List[Any]      )					->				Union[str, Any]:



												'''simple docstring'''

												return datasets.DatasetInfo(features=self.config.features      )




							def 	__snake_case(						self   :					str	,							_UpperCamelCase   :					List[str]      )					->				str:



												'''simple docstring'''

												if not self.config.data_files:
																	raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"      )
												SCREAMING_SNAKE_CASE    			=       dl_manager.download_and_extract(self.config.data_files      )
												if isinstance(UpperCamelCase__	,							(str, list, tuple)      ):
																	SCREAMING_SNAKE_CASE    			=       data_files
																	if isinstance(UpperCamelCase__	,							UpperCamelCase__      ):
																						SCREAMING_SNAKE_CASE    			=       [files]
																	# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																	SCREAMING_SNAKE_CASE    			=       [dl_manager.iter_files(UpperCamelCase__      ) for file in files]
																	return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,							gen_kwargs={"files": files}      )]
												SCREAMING_SNAKE_CASE    			=       []
												for split_name, files in data_files.items():
																	if isinstance(UpperCamelCase__	,							UpperCamelCase__      ):
																						SCREAMING_SNAKE_CASE    			=       [files]
																	# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																	SCREAMING_SNAKE_CASE    			=       [dl_manager.iter_files(UpperCamelCase__      ) for file in files]
																	# Infer features is they are stoed in the arrow schema
																	if self.info.features is None:
																						for file in itertools.chain.from_iterable(UpperCamelCase__      ):
																											with open(UpperCamelCase__	,							"rb"      ) as f:
																																SCREAMING_SNAKE_CASE    			=       datasets.Features.from_arrow_schema(pq.read_schema(UpperCamelCase__      )      )
																											break
																	splits.append(datasets.SplitGenerator(name=UpperCamelCase__	,							gen_kwargs={"files": files}      )      )
												return splits




							def 	__snake_case(						self   :					Union[str, Any]	,							_UpperCamelCase   :					List[str]      )					->				Union[str, Any]:



												'''simple docstring'''

												if self.info.features is not None:
																	# more expensive cast to support nested features with keys in a different order
																	# allows str <-> int/float or str to Audio for example
																	SCREAMING_SNAKE_CASE    			=       table_cast(UpperCamelCase__	,							self.info.features.arrow_schema      )
												return pa_table





							def 	__snake_case(						self   :					List[str]	,							_UpperCamelCase   :					Dict      )					->				str:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       self.info.features.arrow_schema if self.info.features is not None else None
												if self.info.features is not None and self.config.columns is not None:
																	if sorted(field.name for field in schema      ) != sorted(self.config.columns      ):
																						raise ValueError(
																						    F"Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"      )
												for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase__      )      ):
																	with open(UpperCamelCase__	,							"rb"      ) as f:
																						SCREAMING_SNAKE_CASE    			=       pq.ParquetFile(UpperCamelCase__      )
																						try:
																											for batch_idx, record_batch in enumerate(
																											    parquet_file.iter_batches(batch_size=self.config.batch_size	,							columns=self.config.columns      )      ):
																																SCREAMING_SNAKE_CASE    			=       pa.Table.from_batches([record_batch]      )
																																# Uncomment for debugging (will print the Arrow table size and elements)
																																# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																yield F"{file_idx}_{batch_idx}", self._cast_table(UpperCamelCase__      )
																						except ValueError as e:
																											logger.error(F"Failed to read file '{file}' with error {type(UpperCamelCase__      )}: {e}"      )
																											raise





import functools
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

import huggingface_hub.utils as hf_hub_utils
from tqdm import auto as tqdm_lib


_lowerCamelCase					:   str							=		threading.Lock()
_lowerCamelCase					:   Optional[logging.Handler]							=		None

_lowerCamelCase					:   Any							=		{
    '''debug''': logging.DEBUG,
    '''info''': logging.INFO,
    '''warning''': logging.WARNING,
    '''error''': logging.ERROR,
    '''critical''': logging.CRITICAL,
}

_lowerCamelCase					:   Union[str, Any]							=		logging.WARNING

_lowerCamelCase					:   List[Any]							=		True





def 	__lowerCamelCase							():
					SCREAMING_SNAKE_CASE    			=       os.getenv("TRANSFORMERS_VERBOSITY"      , UpperCAmelCase__  )
					if env_level_str:
										if env_level_str in log_levels:
															return log_levels[env_level_str]
										else:
															logging.getLogger().warning(
															    F"Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, "
															    F"has to be one of: { ', '.join(log_levels.keys()  ) }"  )
					return _default_log_level





def 	__lowerCamelCase							():
					return __name__.split("."  )[0]





def 	__lowerCamelCase							():
					return logging.getLogger(_get_library_name()  )





def 	__lowerCamelCase							():
					global _default_handler

					with _lock:
										if _default_handler:
															# This library has already configured the library root logger.
															return
										SCREAMING_SNAKE_CASE    			=       logging.StreamHandler()  # Set sys.stderr as stream.
										SCREAMING_SNAKE_CASE    			=       sys.stderr.flush

										# Apply our default configuration to the library root logger.
										SCREAMING_SNAKE_CASE    			=       _get_library_root_logger()
										library_root_logger.addHandler(_default_handler  )
										library_root_logger.setLevel(_get_default_logging_level()  )
										SCREAMING_SNAKE_CASE    			=       False





def 	__lowerCamelCase							():
					global _default_handler

					with _lock:
										if not _default_handler:
															return

										SCREAMING_SNAKE_CASE    			=       _get_library_root_logger()
										library_root_logger.removeHandler(_default_handler  )
										library_root_logger.setLevel(logging.NOTSET  )
										SCREAMING_SNAKE_CASE    			=       None





def 	__lowerCamelCase							():
					return log_levels





def 	__lowerCamelCase							(UpperCAmelCase__     :							Optional[str] = None  ):

					if name is None:
										SCREAMING_SNAKE_CASE    			=       _get_library_name()

					_configure_library_root_logger()
					return logging.getLogger(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					_configure_library_root_logger()
					return _get_library_root_logger().getEffectiveLevel()





def 	__lowerCamelCase							(UpperCAmelCase__     :							int  ):
					_configure_library_root_logger()
					_get_library_root_logger().setLevel(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					return set_verbosity(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					return set_verbosity(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					return set_verbosity(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					return set_verbosity(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					_configure_library_root_logger()

					assert _default_handler is not None
					_get_library_root_logger().removeHandler(_default_handler  )





def 	__lowerCamelCase							():
					_configure_library_root_logger()

					assert _default_handler is not None
					_get_library_root_logger().addHandler(_default_handler  )





def 	__lowerCamelCase							(UpperCAmelCase__     :							logging.Handler  ):
					_configure_library_root_logger()

					assert handler is not None
					_get_library_root_logger().addHandler(UpperCAmelCase__  )





def 	__lowerCamelCase							(UpperCAmelCase__     :							logging.Handler  ):
					_configure_library_root_logger()

					assert handler is not None and handler not in _get_library_root_logger().handlers
					_get_library_root_logger().removeHandler(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					_configure_library_root_logger()
					SCREAMING_SNAKE_CASE    			=       False





def 	__lowerCamelCase							():
					_configure_library_root_logger()
					SCREAMING_SNAKE_CASE    			=       True





def 	__lowerCamelCase							():
					SCREAMING_SNAKE_CASE    			=       _get_library_root_logger().handlers

					for handler in handlers:
										SCREAMING_SNAKE_CASE    			=       logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s"  )
										handler.setFormatter(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					SCREAMING_SNAKE_CASE    			=       _get_library_root_logger().handlers

					for handler in handlers:
										handler.setFormatter(UpperCAmelCase__  )





def 	__lowerCamelCase							(self     :							str      , *UpperCAmelCase__     :							Union[str, Any]      , **UpperCAmelCase__     :							List[str]  ):
					SCREAMING_SNAKE_CASE    			=       os.getenv("TRANSFORMERS_NO_ADVISORY_WARNINGS"      , UpperCAmelCase__  )
					if no_advisory_warnings:
										return
					self.warning(*UpperCAmelCase__      , **UpperCAmelCase__  )


_lowerCamelCase					:   str							=		warning_advice





@functools.lru_cache(UpperCAmelCase__  )
def 	__lowerCamelCase							(self     :							List[str]      , *UpperCAmelCase__     :							Tuple      , **UpperCAmelCase__     :							int  ):
					self.warning(*UpperCAmelCase__      , **UpperCAmelCase__  )


_lowerCamelCase					:   Dict							=		warning_once


class 						lowercase    :




							def __init__(						self   :					List[Any]	,							*_UpperCamelCase   :					Union[str, Any]	,							**_UpperCamelCase   :					str      )					->				List[Any]:  # pylint: disable=unused-argument



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       args[0] if args else None




							def __iter__(						self   :					Optional[Any]      )					->				str:



												'''simple docstring'''

												return iter(self._iterator      )




							def __getattr__(						self   :					List[str]	,							_UpperCamelCase   :					Any      )					->				List[Any]:



												'''simple docstring'''

												def empty_fn(*_UpperCamelCase   :					List[str]	,							**_UpperCamelCase   :					Union[str, Any]      ):  # pylint: disable=unused-argument
																	return

												return empty_fn




							def __enter__(						self   :					Any      )					->				Optional[Any]:



												'''simple docstring'''

												return self





							def __exit__(						self   :					Optional[Any]	,							_UpperCamelCase   :					Tuple	,							_UpperCamelCase   :					str	,							_UpperCamelCase   :					List[str]      )					->				Union[str, Any]:



												'''simple docstring'''

												return



class 						lowercase    :




							def __call__(						self   :					Union[str, Any]	,							*_UpperCamelCase   :					Optional[Any]	,							**_UpperCamelCase   :					Tuple      )					->				Tuple:



												'''simple docstring'''

												if _tqdm_active:
																	return tqdm_lib.tqdm(*_UpperCamelCase	,							**_UpperCamelCase      )
												else:
																	return EmptyTqdm(*_UpperCamelCase	,							**_UpperCamelCase      )




							def 	__snake_case(						self   :					Dict	,							*_UpperCamelCase   :					Dict	,							**_UpperCamelCase   :					Union[str, Any]      )					->				Union[str, Any]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       None
												if _tqdm_active:
																	return tqdm_lib.tqdm.set_lock(*_UpperCamelCase	,							**_UpperCamelCase      )





							def 	__snake_case(						self   :					Optional[int]      )					->				Union[str, Any]:



												'''simple docstring'''

												if _tqdm_active:
																	return tqdm_lib.tqdm.get_lock()



_lowerCamelCase					:   Union[str, Any]							=		_tqdm_cls()





def 	__lowerCamelCase							():
					global _tqdm_active
					return bool(_tqdm_active  )





def 	__lowerCamelCase							():
					global _tqdm_active
					SCREAMING_SNAKE_CASE    			=       True
					hf_hub_utils.enable_progress_bars()





def 	__lowerCamelCase							():
					global _tqdm_active
					SCREAMING_SNAKE_CASE    			=       False
					hf_hub_utils.disable_progress_bars()




