






import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


lowerCamelCase : Tuple						   =       datasets.utils.logging.get_logger(__name__)

@dataclass
class 		lowerCAmelCase     ( datasets.BuilderConfig			):

					'''simple docstring'''

					_A						:			int									=      10000
					_A						:			Optional[List[str]]									=      None
					_A						:			Optional[datasets.Features]									=      None







class 		lowerCAmelCase     ( datasets.ArrowBasedBuilder			):

					'''simple docstring'''

					_A						:			Dict									=      ParquetConfig



					def       lowerCAmelCase				(  self				:	Optional[Any]							)			->  str:


												"""simple docstring"""

												return datasets.DatasetInfo(features=self.config.features							)



					def       lowerCAmelCase				(  self				:	Union[str, Any]		,     __a				:	Union[str, Any]							)			->  str:


												"""simple docstring"""

												if not self.config.data_files:
																			raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"							)
												__lowercase      :							Optional[Any]				=     dl_manager.download_and_extract(self.config.data_files							)
												if isinstance(__a		,     (str, list, tuple)							):
																			__lowercase      :							Any				=     data_files
																			if isinstance(__a		,     __a							):
																										__lowercase      :							Optional[Any]				=     [files]
																			# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																			__lowercase      :							Any				=     [dl_manager.iter_files(__a							) for file in files]
																			return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,     gen_kwargs={"""files""": files}							)]
												__lowercase      :							Any				=     []
												for split_name, files in data_files.items():
																			if isinstance(__a		,     __a							):
																										__lowercase      :							str				=     [files]
																			# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																			__lowercase      :							str				=     [dl_manager.iter_files(__a							) for file in files]
																			# Infer features is they are stoed in the arrow schema
																			if self.info.features is None:
																										for file in itertools.chain.from_iterable(__a							):
																																	with open(__a		,     """rb"""							) as f:
																																								__lowercase      :							Optional[int]				=     datasets.Features.from_arrow_schema(pq.read_schema(__a							)							)
																																	break
																			splits.append(datasets.SplitGenerator(name=__a		,     gen_kwargs={"""files""": files}							)							)
												return splits



					def       lowerCAmelCase				(  self				:	Dict		,     __a				:	pa.Table							)			->  pa.Table:


												"""simple docstring"""

												if self.info.features is not None:
																			# more expensive cast to support nested features with keys in a different order
																			# allows str <-> int/float or str to Audio for example
																			__lowercase      :							Dict				=     table_cast(__a		,     self.info.features.arrow_schema							)
												return pa_table



					def       lowerCAmelCase				(  self				:	int		,     __a				:	Any							)			->  int:


												"""simple docstring"""

												__lowercase      :							Tuple				=     self.info.features.arrow_schema if self.info.features is not None else None
												if self.info.features is not None and self.config.columns is not None:
																			if sorted(field.name for field in schema							) != sorted(self.config.columns							):
																										raise ValueError(
																										    F"Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"							)
												for file_idx, file in enumerate(itertools.chain.from_iterable(__a							)							):
																			with open(__a		,     """rb"""							) as f:
																										__lowercase      :							str				=     pq.ParquetFile(__a							)
																										try:
																																	for batch_idx, record_batch in enumerate(
																																	    parquet_file.iter_batches(batch_size=self.config.batch_size		,     columns=self.config.columns							)							):
																																								__lowercase      :							Union[str, Any]				=     pa.Table.from_batches([record_batch]							)
																																								# Uncomment for debugging (will print the Arrow table size and elements)
																																								# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																								# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																								yield F"{file_idx}_{batch_idx}", self._cast_table(__a							)
																										except ValueError as e:
																																	logger.error(F"Failed to read file '{file}' with error {type(__a							)}: {e}"							)
																																	raise







import logging
import os
import threading
import time


try:
	import warnings
except ImportError:
	lowerCamelCase : Any						   =       None

try:
	import msvcrt
except ImportError:
	lowerCamelCase : str						   =       None

try:
	import fcntl
except ImportError:
	lowerCamelCase : Optional[Any]						   =       None


# Backward compatibility
# ------------------------------------------------
try:
	TimeoutError
except NameError:
	lowerCamelCase : Union[str, Any]						   =       OSError


# Data
# ------------------------------------------------
lowerCamelCase : Tuple						   =       [
    '''Timeout''',
    '''BaseFileLock''',
    '''WindowsFileLock''',
    '''UnixFileLock''',
    '''SoftFileLock''',
    '''FileLock''',
]

lowerCamelCase : Tuple						   =       '''3.0.12'''


lowerCamelCase : Any						   =       None
def       snake_case_		(     ):
							global _logger
							__lowercase      :							List[str]				=     _logger or logging.getLogger(__name__      )
							return _logger

class 		lowerCAmelCase     ( __a			):

					'''simple docstring'''

					def __init__(  self				:	Any		,     __a				:	Any							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							List[str]				=     lock_file
												return None



					def __str__(  self				:	str							)			->  Any:


												"""simple docstring"""

												__lowercase      :							Any				=     F"The file lock '{self.lock_file}' could not be acquired."
												return temp







class 		lowerCAmelCase     :

					'''simple docstring'''

					def __init__(  self				:	List[Any]		,     __a				:	Optional[int]							)			->  int:


												"""simple docstring"""

												__lowercase      :							Optional[Any]				=     lock
												return None



					def __enter__(  self				:	Dict							)			->  Dict:


												"""simple docstring"""

												return self.lock



					def __exit__(  self				:	Optional[int]		,     __a				:	Dict		,     __a				:	Any		,     __a				:	Tuple							)			->  Optional[Any]:


												"""simple docstring"""

												self.lock.release()
												return None







class 		lowerCAmelCase     :

					'''simple docstring'''

					def __init__(  self				:	Tuple		,     __a				:	Any		,     __a				:	Dict=-1		,     __a				:	Optional[Any]=None							)			->  Any:


												"""simple docstring"""

												__lowercase      :							Optional[int]				=     max_filename_length if max_filename_length is not None else 255
												# Hash the filename if it's too long
												__lowercase      :							Dict				=     self.hash_filename_if_too_long(__a		,     __a							)
												# The path to the lock file.
												__lowercase      :							Optional[Any]				=     lock_file

												# The file descriptor for the *_lock_file* as it is returned by the
												# os.open() function.
												# This file lock is only NOT None, if the object currently holds the
												# lock.
												__lowercase      :							int				=     None

												# The default timeout value.
												__lowercase      :							Optional[int]				=     timeout

												# We use this lock primarily for the lock counter.
												__lowercase      :							Optional[Any]				=     threading.Lock()

												# The lock counter is used for implementing the nested locking
												# mechanism. Whenever the lock is acquired, the counter is increased and
												# the lock is only released, when this value is 0 again.
												__lowercase      :							Union[str, Any]				=     0
												return None



					@property
					def       lowerCAmelCase				(  self				:	Optional[Any]							)			->  Optional[int]:


												"""simple docstring"""

												return self._lock_file



					@property
					def       lowerCAmelCase				(  self				:	Optional[int]							)			->  Optional[Any]:


												"""simple docstring"""

												return self._timeout



					@timeout.setter
					def       lowerCAmelCase				(  self				:	Tuple		,     __a				:	Tuple							)			->  Dict:


												"""simple docstring"""

												__lowercase      :							Tuple				=     float(__a							)
												return None



					def       lowerCAmelCase				(  self				:	Dict							)			->  Tuple:


												"""simple docstring"""

												raise NotImplementedError()



					def       lowerCAmelCase				(  self				:	Dict							)			->  Any:


												"""simple docstring"""

												raise NotImplementedError()



					@property
					def       lowerCAmelCase				(  self				:	Dict							)			->  List[Any]:


												"""simple docstring"""

												return self._lock_file_fd is not None



					def       lowerCAmelCase				(  self				:	Any		,     __a				:	Optional[Any]=None		,     __a				:	Union[str, Any]=0.05							)			->  List[str]:


												"""simple docstring"""

												if timeout is None:
																			__lowercase      :							Union[str, Any]				=     self.timeout

												# Increment the number right at the beginning.
												# We can still undo it, if something fails.
												with self._thread_lock:
																			self._lock_counter += 1

												__lowercase      :							int				=     id(self							)
												__lowercase      :							Optional[Any]				=     self._lock_file
												__lowercase      :							List[str]				=     time.time()
												try:
																			while True:
																										with self._thread_lock:
																																	if not self.is_locked:
																																								logger().debug(F"Attempting to acquire lock {lock_id} on {lock_filename}"							)
																																								self._acquire()

																										if self.is_locked:
																																	logger().debug(F"Lock {lock_id} acquired on {lock_filename}"							)
																																	break
																										elif timeout >= 0 and time.time() - start_time > timeout:
																																	logger().debug(F"Timeout on acquiring lock {lock_id} on {lock_filename}"							)
																																	raise Timeout(self._lock_file							)
																										else:
																																	logger().debug(
																																	    F"Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ..."							)
																																	time.sleep(__a							)
												except:  # noqa
																			# Something did go wrong, so decrement the counter.
																			with self._thread_lock:
																										__lowercase      :							Optional[int]				=     max(0		,     self._lock_counter - 1							)

																			raise
												return _Acquire_ReturnProxy(lock=self							)



					def       lowerCAmelCase				(  self				:	Union[str, Any]		,     __a				:	Optional[Any]=False							)			->  Optional[Any]:


												"""simple docstring"""

												with self._thread_lock:
																			if self.is_locked:
																										self._lock_counter -= 1

																										if self._lock_counter == 0 or force:
																																	__lowercase      :							Optional[Any]				=     id(self							)
																																	__lowercase      :							str				=     self._lock_file

																																	logger().debug(F"Attempting to release lock {lock_id} on {lock_filename}"							)
																																	self._release()
																																	__lowercase      :							List[str]				=     0
																																	logger().debug(F"Lock {lock_id} released on {lock_filename}"							)

												return None



					def __enter__(  self				:	Any							)			->  Optional[Any]:


												"""simple docstring"""

												self.acquire()
												return self



					def __exit__(  self				:	List[str]		,     __a				:	str		,     __a				:	int		,     __a				:	List[Any]							)			->  Tuple:


												"""simple docstring"""

												self.release()
												return None



					def __del__(  self				:	Optional[int]							)			->  Optional[int]:


												"""simple docstring"""

												self.release(force=__a							)
												return None



					def       lowerCAmelCase				(  self				:	Tuple		,     __a				:	str		,     __a				:	int							)			->  str:


												"""simple docstring"""

												__lowercase      :							List[Any]				=     os.path.basename(__a							)
												if len(__a							) > max_length and max_length > 0:
																			__lowercase      :							int				=     os.path.dirname(__a							)
																			__lowercase      :							List[str]				=     str(hash(__a							)							)
																			__lowercase      :							Optional[Any]				=     filename[: max_length - len(__a							) - 8] + """...""" + hashed_filename + """.lock"""
																			return os.path.join(__a		,     __a							)
												else:
																			return path







class 		lowerCAmelCase     ( __a			):

					'''simple docstring'''

					def __init__(  self				:	Union[str, Any]		,     __a				:	List[Any]		,     __a				:	Optional[int]=-1		,     __a				:	Tuple=None							)			->  List[Any]:


												"""simple docstring"""

												from .file_utils import relative_to_absolute_path

												super().__init__(__a		,     timeout=__a		,     max_filename_length=__a							)
												__lowercase      :							Tuple				=     """\\\\?\\""" + relative_to_absolute_path(self.lock_file							)



					def       lowerCAmelCase				(  self				:	Dict							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							List[str]				=     os.O_RDWR | os.O_CREAT | os.O_TRUNC

												try:
																			__lowercase      :							Tuple				=     os.open(self._lock_file		,     __a							)
												except OSError:
																			pass
												else:
																			try:
																										msvcrt.locking(__a		,     msvcrt.LK_NBLCK		,     1							)
																			except OSError:
																										os.close(__a							)
																			else:
																										__lowercase      :							Union[str, Any]				=     fd
												return None



					def       lowerCAmelCase				(  self				:	Any							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							Optional[Any]				=     self._lock_file_fd
												__lowercase      :							int				=     None
												msvcrt.locking(__a		,     msvcrt.LK_UNLCK		,     1							)
												os.close(__a							)

												try:
																			os.remove(self._lock_file							)
												# Probably another instance of the application
												# that acquired the file lock.
												except OSError:
																			pass
												return None







class 		lowerCAmelCase     ( __a			):

					'''simple docstring'''

					def __init__(  self				:	List[str]		,     __a				:	Optional[Any]		,     __a				:	str=-1		,     __a				:	List[str]=None							)			->  Any:


												"""simple docstring"""

												__lowercase      :							Dict				=     os.statvfs(os.path.dirname(__a							)							).f_namemax
												super().__init__(__a		,     timeout=__a		,     max_filename_length=__a							)



					def       lowerCAmelCase				(  self				:	Optional[Any]							)			->  Dict:


												"""simple docstring"""

												__lowercase      :							List[Any]				=     os.O_RDWR | os.O_CREAT | os.O_TRUNC
												__lowercase      :							List[str]				=     os.open(self._lock_file		,     __a							)

												try:
																			fcntl.flock(__a		,     fcntl.LOCK_EX | fcntl.LOCK_NB							)
												except OSError:
																			os.close(__a							)
												else:
																			__lowercase      :							str				=     fd
												return None



					def       lowerCAmelCase				(  self				:	List[Any]							)			->  Optional[int]:


												"""simple docstring"""

												__lowercase      :							Any				=     self._lock_file_fd
												__lowercase      :							List[str]				=     None
												fcntl.flock(__a		,     fcntl.LOCK_UN							)
												os.close(__a							)
												return None







class 		lowerCAmelCase     ( __a			):

					'''simple docstring'''

					def       lowerCAmelCase				(  self				:	List[str]							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							Tuple				=     os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
												try:
																			__lowercase      :							Union[str, Any]				=     os.open(self._lock_file		,     __a							)
												except OSError:
																			pass
												else:
																			__lowercase      :							Optional[int]				=     fd
												return None



					def       lowerCAmelCase				(  self				:	List[str]							)			->  Tuple:


												"""simple docstring"""

												os.close(self._lock_file_fd							)
												__lowercase      :							int				=     None

												try:
																			os.remove(self._lock_file							)
												# The file is already deleted and that's what we want.
												except OSError:
																			pass
												return None







lowerCamelCase : Optional[Any]						   =       None

if msvcrt:
	lowerCamelCase : List[Any]						   =       WindowsFileLock
elif fcntl:
	lowerCamelCase : List[Any]						   =       UnixFileLock
else:
	lowerCamelCase : Union[str, Any]						   =       SoftFileLock

	if warnings is not None:
		warnings.warn('''only soft file lock is available''')