





'''simple docstring'''




_A:					Tuple												=				"""0.21.0"""

from .accelerator import Accelerator
from .big_modeling import (
    cpu_offload,
    cpu_offload_with_hook,
    disk_offload,
    dispatch_model,
    init_empty_weights,
    init_on_device,
    load_checkpoint_and_dispatch,
)
from .data_loader import skip_first_batches
from .launchers import debug_launcher, notebook_launcher
from .state import PartialState
from .utils import (
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    FullyShardedDataParallelPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    find_executable_batch_size,
    infer_auto_device_map,
    is_rich_available,
    load_checkpoint_in_model,
    synchronize_rng_states,
)


if is_rich_available():
    from .utils import rich












'''simple docstring'''




import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
    import sqlitea

    import sqlalchemy






class UpperCAmelCase							(		UpperCAmelCase_							):



    def __init__(  self     ,			__A     ,			__A     ,			__A = None     ,			__A = None     ,			__A = False     ,			**__A     ,			):
     super().__init__(features=__A     ,			cache_dir=__A     ,			keep_in_memory=__A     ,			**__A					)
     __UpperCAmelCase					=   Sql(
         cache_dir=__A     ,			features=__A     ,			sql=__A     ,			con=__A     ,			**__A     ,			)




    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   None
     __UpperCAmelCase					=   None
     __UpperCAmelCase					=   None
     __UpperCAmelCase					=   None

     self.builder.download_and_prepare(
         download_config=__A     ,			download_mode=__A     ,			verification_mode=__A     ,			base_path=__A     ,			)

     # Build dataset for splits
     __UpperCAmelCase					=   self.builder.as_dataset(
         split='train'     ,			verification_mode=__A     ,			in_memory=self.keep_in_memory					)
     return dataset






class UpperCAmelCase							:



    def __init__(  self     ,			__A     ,			__A     ,			__A     ,			__A = None     ,			__A = None     ,			**__A     ,			):
     if num_proc is not None and num_proc <= 0:
      raise ValueError(f'num_proc {num_proc} must be an integer > 0.'					)

     __UpperCAmelCase					=   dataset
     __UpperCAmelCase					=   name
     __UpperCAmelCase					=   con
     __UpperCAmelCase					=   batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
     __UpperCAmelCase					=   num_proc
     __UpperCAmelCase					=   to_sql_kwargs



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   self.to_sql_kwargs.pop('sql'     ,			__A					)
     __UpperCAmelCase					=   self.to_sql_kwargs.pop('con'     ,			__A					)
     __UpperCAmelCase					=   self.to_sql_kwargs.pop('index'     ,			__A					)

     __UpperCAmelCase					=   self._write(index=__A     ,			**self.to_sql_kwargs					)
     return written



    def 					__lowerCamelCase						(  self     ,			__A					):
     __UpperCAmelCase					,    __UpperCAmelCase					,    __UpperCAmelCase					=   args
     __UpperCAmelCase					=   {**to_sql_kwargs, 'if_exists': 'append'} if offset > 0 else to_sql_kwargs
     __UpperCAmelCase					=   query_table(
         table=self.dataset.data     ,			key=slice(__A     ,			offset + self.batch_size					)     ,			indices=self.dataset._indices     ,			)
     __UpperCAmelCase					=   batch.to_pandas()
     __UpperCAmelCase					=   df.to_sql(self.name     ,			self.con     ,			index=__A     ,			**__A					)
     return num_rows or len(__A					)




    def 					__lowerCamelCase						(  self     ,			__A     ,			**__A					):
     __UpperCAmelCase					=   0

     if self.num_proc is None or self.num_proc == 1:
      for offset in logging.tqdm(
          range(0     ,			len(self.dataset					)     ,			self.batch_size					)     ,			unit='ba'     ,			disable=not logging.is_progress_bar_enabled()     ,			desc='Creating SQL from Arrow format'     ,			):
       written += self._batch_sql((offset, index, to_sql_kwargs)					)
     else:
      __UpperCAmelCase					,    __UpperCAmelCase					=   len(self.dataset					), self.batch_size
      with multiprocessing.Pool(self.num_proc					) as pool:
       for num_rows in logging.tqdm(
           pool.imap(
               self._batch_sql     ,			[(offset, index, to_sql_kwargs) for offset in range(0     ,			__A     ,			__A					)]     ,			)     ,			total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size     ,			unit='ba'     ,			disable=not logging.is_progress_bar_enabled()     ,			desc='Creating SQL from Arrow format'     ,			):
        written += num_rows

     return written





