


'''simple docstring'''







import os
import posixpath
import uuid
from dataclasses import dataclass
from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, Union

import numpy as np
import pyarrow as pa

import datasets
from datasets.arrow_writer import ArrowWriter, ParquetWriter
from datasets.config import MAX_SHARD_SIZE
from datasets.filesystems import (
    is_remote_filesystem,
    rename,
)
from datasets.iterable_dataset import _BaseExamplesIterable
from datasets.utils.py_utils import convert_file_size_to_int


lowerCAmelCase    :     Union[str, Any]					=						datasets.utils.logging.get_logger(__name__)

if TYPE_CHECKING:
       import pyspark
@dataclass
class 	UpperCamelCase__					( datasets.BuilderConfig	):



     """simple docstring"""






     __magic_name__    			=    None






def 			lowercase				(_A   ,						_A   ,						):







       """simple docstring"""







       import pyspark

       def generate_fn():
              _lowerCAmelCase    :							List[Any]   			=       df.select('*'   ,						pyspark.sql.functions.spark_partition_id().alias('part_id'					)					)
              for partition_id in partition_order:
                     _lowerCAmelCase    :							str   			=       df_with_partition_id.select('*'					).where(f'part_id = {partition_id}'					).drop('part_id'					)
                     _lowerCAmelCase    :							int   			=       partition_df.collect()
                     _lowerCAmelCase    :							str   			=       0
                     for row in rows:
                            yield f'{partition_id}_{row_id}', row.asDict()
                            row_id += 1

       return generate_fn
class 	UpperCamelCase__					( _BaseExamplesIterable	):



     """simple docstring"""






     def __init__( self      ,	snake_case__      ,	snake_case__=None      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							List[str]   			=       df
            _lowerCAmelCase    :							str   			=       partition_order or range(self.df.rdd.getNumPartitions()   )
            _lowerCAmelCase    :							Dict   			=       _generate_iterable_examples(self.df      ,	self.partition_order   )







     def __iter__( self   ):

            '''simple docstring'''

            yield from self.generate_examples_fn()







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							List[Any]   			=       list(range(self.df.rdd.getNumPartitions()   )   )
            generator.shuffle(snake_case__   )
            return SparkExamplesIterable(self.df      ,	partition_order=snake_case__   )







     def a	( self      ,	snake_case__      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[Any]   			=       self.split_shard_indices_by_worker(snake_case__      ,	snake_case__   )
            return SparkExamplesIterable(self.df      ,	partition_order=snake_case__   )







     @property
     def a	( self   ):

            '''simple docstring'''

            return len(self.partition_order   )
class 	UpperCamelCase__					( datasets.DatasetBuilder	):



     """simple docstring"""






     __magic_name__    			=    SparkConfig







     def __init__( self      ,	snake_case__      ,	snake_case__ = None      ,	snake_case__ = None      ,	**snake_case__      ,	):

            '''simple docstring'''

            import pyspark

            _lowerCAmelCase    :							Dict   			=       pyspark.sql.SparkSession.builder.getOrCreate()
            _lowerCAmelCase    :							List[str]   			=       df
            _lowerCAmelCase    :							str   			=       working_dir

            super().__init__(
                cache_dir=snake_case__      ,	config_name=str(self.df.semanticHash()   )      ,	**snake_case__      ,	)







     def a	( self   ):

            '''simple docstring'''

            def create_cache_and_write_probe(snake_case__   ):
                   # makedirs with exist_ok will recursively create the directory. It will not throw an error if directories
                   # already exist.
                   os.makedirs(self._cache_dir      ,	exist_ok=snake_case__   )
                   _lowerCAmelCase    :							Union[str, Any]   			=       os.path.join(self._cache_dir      ,	'fs_test' + uuid.uuida().hex   )
                   # Opening the file in append mode will create a new file unless it already exists, in which case it will not
                   # change the file contents.
                   open(snake_case__      ,	'a'   )
                   return [probe_file]

            if self._spark.conf.get('spark.master'      ,	''   ).startswith('local'   ):
                   return

            # If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS
            # accessible to the driver.
            # TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error.
            if self._cache_dir:
                   _lowerCAmelCase    :							Union[str, Any]   			=       (
                       self._spark.sparkContext.parallelize(range(1   )      ,	1   ).mapPartitions(snake_case__   ).collect()
                   )
                   if os.path.isfile(probe[0]   ):
                          return

            raise ValueError(
                'When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir'   )







     def a	( self   ):

            '''simple docstring'''

            return datasets.DatasetInfo(features=self.config.features   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            return [datasets.SplitGenerator(name=datasets.Split.TRAIN   )]







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            import pyspark

            def get_arrow_batch_size(snake_case__   ):
                   for batch in it:
                          yield pa.RecordBatch.from_pydict({'batch_bytes': [batch.nbytes]}   )

            _lowerCAmelCase    :							Tuple   			=       self.df.count()
            _lowerCAmelCase    :							int   			=       df_num_rows if df_num_rows <= 100 else 100
            # Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample.
            _lowerCAmelCase    :							Dict   			=       (
                self.df.limit(snake_case__   )
                .repartition(1   )
                .mapInArrow(snake_case__      ,	'batch_bytes: long'   )
                .agg(pyspark.sql.functions.sum('batch_bytes'   ).alias('sample_bytes'   )   )
                .collect()[0]
                .sample_bytes
                / sample_num_rows
            )
            _lowerCAmelCase    :							List[str]   			=       approx_bytes_per_row * df_num_rows
            if approx_total_size > max_shard_size:
                   # Make sure there is at least one row per partition.
                   _lowerCAmelCase    :							List[str]   			=       min(snake_case__      ,	int(approx_total_size / max_shard_size   )   )
                   _lowerCAmelCase    :							Any   			=       self.df.repartition(snake_case__   )







     def a	( self      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	):

            '''simple docstring'''

            import pyspark

            _lowerCAmelCase    :							Optional[Any]   			=       ParquetWriter if file_format == 'parquet' else ArrowWriter
            _lowerCAmelCase    :							int   			=       os.path.join(self._working_dir      ,	os.path.basename(snake_case__   )   ) if self._working_dir else fpath
            _lowerCAmelCase    :							List[str]   			=       file_format == 'parquet'

            # Define these so that we don't reference self in write_arrow, which will result in a pickling error due to
            # pickling the SparkContext.
            _lowerCAmelCase    :							List[str]   			=       self.config.features
            _lowerCAmelCase    :							str   			=       self._writer_batch_size
            _lowerCAmelCase    :							Union[str, Any]   			=       self._fs.storage_options

            def write_arrow(snake_case__   ):
                   # Within the same SparkContext, no two task attempts will share the same attempt ID.
                   _lowerCAmelCase    :							Tuple   			=       pyspark.TaskContext().taskAttemptId()
                   _lowerCAmelCase    :							Union[str, Any]   			=       next(snake_case__      ,	snake_case__   )
                   if first_batch is None:
                          # Some partitions might not receive any data.
                          return pa.RecordBatch.from_arrays(
                              [[task_id], [0], [0]]      ,	names=['task_id', 'num_examples', 'num_bytes']      ,	)
                   _lowerCAmelCase    :							Optional[Any]   			=       0
                   _lowerCAmelCase    :							Tuple   			=       writer_class(
                       features=snake_case__      ,	path=working_fpath.replace('SSSSS'      ,	F'{shard_id:05d}'   ).replace('TTTTT'      ,	F'{task_id:05d}'   )      ,	writer_batch_size=snake_case__      ,	storage_options=snake_case__      ,	embed_local_files=snake_case__      ,	)
                   _lowerCAmelCase    :							Union[str, Any]   			=       pa.Table.from_batches([first_batch]   )
                   writer.write_table(snake_case__   )
                   for batch in it:
                          if max_shard_size is not None and writer._num_bytes >= max_shard_size:
                                 _lowerCAmelCase			,     _lowerCAmelCase    :							List[str]   			=       writer.finalize()
                                 writer.close()
                                 yield pa.RecordBatch.from_arrays(
                                     [[task_id], [num_examples], [num_bytes]]      ,	names=['task_id', 'num_examples', 'num_bytes']      ,	)
                                 shard_id += 1
                                 _lowerCAmelCase    :							Union[str, Any]   			=       writer_class(
                                     features=writer._features      ,	path=working_fpath.replace('SSSSS'      ,	F'{shard_id:05d}'   ).replace('TTTTT'      ,	F'{task_id:05d}'   )      ,	writer_batch_size=snake_case__      ,	storage_options=snake_case__      ,	embed_local_files=snake_case__      ,	)
                          _lowerCAmelCase    :							Optional[Any]   			=       pa.Table.from_batches([batch]   )
                          writer.write_table(snake_case__   )

                   if writer._num_bytes > 0:
                          _lowerCAmelCase			,     _lowerCAmelCase    :							int   			=       writer.finalize()
                          writer.close()
                          yield pa.RecordBatch.from_arrays(
                              [[task_id], [num_examples], [num_bytes]]      ,	names=['task_id', 'num_examples', 'num_bytes']      ,	)

                   if working_fpath != fpath:
                          for file in os.listdir(os.path.dirname(snake_case__   )   ):
                                 _lowerCAmelCase    :							Any   			=       os.path.join(os.path.dirname(snake_case__   )      ,	os.path.basename(snake_case__   )   )
                                 shutil.move(snake_case__      ,	snake_case__   )

            _lowerCAmelCase    :							Optional[int]   			=       (
                self.df.mapInArrow(snake_case__      ,	'task_id: long, num_examples: long, num_bytes: long'   )
                .groupBy('task_id'   )
                .agg(
                    pyspark.sql.functions.sum('num_examples'   ).alias('total_num_examples'   )      ,	pyspark.sql.functions.sum('num_bytes'   ).alias('total_num_bytes'   )      ,	pyspark.sql.functions.count('num_bytes'   ).alias('num_shards'   )      ,	pyspark.sql.functions.collect_list('num_examples'   ).alias('shard_lengths'   )      ,	)
                .collect()
            )
            for row in stats:
                   yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths)







     def a	( self      ,	snake_case__      ,	snake_case__ = "arrow"      ,	snake_case__ = None      ,	snake_case__ = None      ,	**snake_case__      ,	):

            '''simple docstring'''

            self._validate_cache_dir()

            _lowerCAmelCase    :							str   			=       convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE   )
            self._repartition_df_if_needed(snake_case__   )
            _lowerCAmelCase    :							Union[str, Any]   			=       not is_remote_filesystem(self._fs   )
            _lowerCAmelCase    :							Tuple   			=       os.path.join if is_local else posixpath.join

            _lowerCAmelCase    :							List[Any]   			=       '-TTTTT-SSSSS-of-NNNNN'
            _lowerCAmelCase    :							Dict   			=       F'{self.name}-{split_generator.name}{SUFFIX}.{file_format}'
            _lowerCAmelCase    :							Optional[Any]   			=       path_join(self._output_dir      ,	snake_case__   )

            _lowerCAmelCase    :							Union[str, Any]   			=       0
            _lowerCAmelCase    :							Tuple   			=       0
            _lowerCAmelCase    :							Union[str, Any]   			=       0
            _lowerCAmelCase    :							Optional[Any]   			=       []
            _lowerCAmelCase    :							List[str]   			=       []

            for task_id, content in self._prepare_split_single(snake_case__      ,	snake_case__      ,	snake_case__   ):
                   (
                       (
                       _lowerCAmelCase
                   )			,     (
                       _lowerCAmelCase
                   )			,     (
                       _lowerCAmelCase
                   )			,     (
                       _lowerCAmelCase
                   )			,     
                   )    :							str   			=       content
                   if num_bytes > 0:
                          total_num_examples += num_examples
                          total_num_bytes += num_bytes
                          total_shards += num_shards
                          task_id_and_num_shards.append((task_id, num_shards)   )
                          all_shard_lengths.extend(snake_case__   )

            _lowerCAmelCase    :							List[Any]   			=       total_num_examples
            _lowerCAmelCase    :							List[Any]   			=       total_num_bytes

            # should rename everything at the end
            logger.debug(F'Renaming {total_shards} shards.'   )
            if total_shards > 1:
                   _lowerCAmelCase    :							Dict   			=       all_shard_lengths

                   # Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a
                   # pickling error due to pickling the SparkContext.
                   _lowerCAmelCase    :							int   			=       self._fs

                   # use the -SSSSS-of-NNNNN pattern
                   def _rename_shard(
                       snake_case__      ,	snake_case__      ,	snake_case__      ,	):
                          rename(
                              snake_case__      ,	fpath.replace('SSSSS'      ,	F'{shard_id:05d}'   ).replace('TTTTT'      ,	F'{task_id:05d}'   )      ,	fpath.replace('TTTTT-SSSSS'      ,	F'{global_shard_id:05d}'   ).replace('NNNNN'      ,	F'{total_shards:05d}'   )      ,	)

                   _lowerCAmelCase    :							int   			=       []
                   _lowerCAmelCase    :							Tuple   			=       0
                   for i in range(len(snake_case__   )   ):
                          _lowerCAmelCase			,     _lowerCAmelCase    :							Any   			=       task_id_and_num_shards[i]
                          for shard_id in range(snake_case__   ):
                                 args.append([task_id, shard_id, global_shard_id]   )
                                 global_shard_id += 1
                   self._spark.sparkContext.parallelize(snake_case__      ,	len(snake_case__   )   ).map(lambda snake_case__   : _rename_shard(*snake_case__   )   ).collect()
            else:
                   # don't use any pattern
                   _lowerCAmelCase    :							Optional[Any]   			=       0
                   _lowerCAmelCase    :							Optional[Any]   			=       task_id_and_num_shards[0][0]
                   self._rename(
                       fpath.replace('SSSSS'      ,	F'{shard_id:05d}'   ).replace('TTTTT'      ,	F'{task_id:05d}'   )      ,	fpath.replace(snake_case__      ,	''   )      ,	)







     def a	( self      ,	snake_case__      ,	):

            '''simple docstring'''

            return SparkExamplesIterable(self.df   )





'''simple docstring'''







from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowerCAmelCase    :     Optional[Any]					=						logging.get_logger(__name__)

lowerCAmelCase    :     str					=						{
    """EleutherAI/gpt-j-6B""": """https://huggingface.co/EleutherAI/gpt-j-6B/resolve/main/config.json""",
    # See all GPT-J models at https://huggingface.co/models?filter=gpt_j
}
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     __magic_name__    			=    "gptj"
     __magic_name__    			=    {
         "max_position_embeddings": "n_positions",
         "hidden_size": "n_embd",
         "num_attention_heads": "n_head",
         "num_hidden_layers": "n_layer",
     }







     def __init__( self      ,	snake_case__=5_0400      ,	snake_case__=2048      ,	snake_case__=4096      ,	snake_case__=28      ,	snake_case__=16      ,	snake_case__=64      ,	snake_case__=None      ,	snake_case__="gelu_new"      ,	snake_case__=0.0      ,	snake_case__=0.0      ,	snake_case__=0.0      ,	snake_case__=1E-5      ,	snake_case__=0.02      ,	snake_case__=True      ,	snake_case__=5_0256      ,	snake_case__=5_0256      ,	snake_case__=False      ,	**snake_case__      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							int   			=       vocab_size
            _lowerCAmelCase    :							Optional[int]   			=       n_positions
            _lowerCAmelCase    :							Optional[int]   			=       n_embd
            _lowerCAmelCase    :							Optional[int]   			=       n_layer
            _lowerCAmelCase    :							str   			=       n_head
            _lowerCAmelCase    :							Tuple   			=       n_inner
            _lowerCAmelCase    :							Tuple   			=       rotary_dim
            _lowerCAmelCase    :							Optional[int]   			=       activation_function
            _lowerCAmelCase    :							Any   			=       resid_pdrop
            _lowerCAmelCase    :							List[str]   			=       embd_pdrop
            _lowerCAmelCase    :							int   			=       attn_pdrop
            _lowerCAmelCase    :							Any   			=       layer_norm_epsilon
            _lowerCAmelCase    :							Optional[int]   			=       initializer_range
            _lowerCAmelCase    :							List[str]   			=       use_cache

            _lowerCAmelCase    :							Dict   			=       bos_token_id
            _lowerCAmelCase    :							Any   			=       eos_token_id

            super().__init__(
                bos_token_id=snake_case__      ,	eos_token_id=snake_case__      ,	tie_word_embeddings=snake_case__      ,	**snake_case__   )
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     def __init__( self      ,	snake_case__      ,	snake_case__ = "default"      ,	snake_case__ = None      ,	snake_case__ = False      ,	):

            '''simple docstring'''

            super().__init__(snake_case__      ,	task=snake_case__      ,	patching_specs=snake_case__      ,	use_past=snake_case__   )
            if not getattr(self._config      ,	'pad_token_id'      ,	snake_case__   ):
                   # TODO: how to do that better?
                   _lowerCAmelCase    :							Any   			=       0







     @property
     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							str   			=       OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}   )
            if self.use_past:
                   self.fill_with_past_key_values_(snake_case__      ,	direction='inputs'   )
                   _lowerCAmelCase    :							int   			=       {0: 'batch', 1: 'past_sequence + sequence'}
            else:
                   _lowerCAmelCase    :							int   			=       {0: 'batch', 1: 'sequence'}

            return common_inputs







     @property
     def a	( self   ):

            '''simple docstring'''

            return self._config.n_layer







     @property
     def a	( self   ):

            '''simple docstring'''

            return self._config.n_head







     def a	( self      ,	snake_case__      ,	snake_case__ = -1      ,	snake_case__ = -1      ,	snake_case__ = False      ,	snake_case__ = None      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[Any]   			=       super(snake_case__      ,	self   ).generate_dummy_inputs(
                snake_case__      ,	batch_size=snake_case__      ,	seq_length=snake_case__      ,	is_pair=snake_case__      ,	framework=snake_case__   )

            # We need to order the input in the way they appears in the forward()
            _lowerCAmelCase    :							Any   			=       OrderedDict({'input_ids': common_inputs['input_ids']}   )

            # Need to add the past_keys
            if self.use_past:
                   if not is_torch_available():
                          raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'   )
                   else:
                          import torch

                          _lowerCAmelCase			,     _lowerCAmelCase    :							Optional[Any]   			=       common_inputs['input_ids'].shape
                          # Not using the same length for past_key_values
                          _lowerCAmelCase    :							Any   			=       seqlen + 2
                          _lowerCAmelCase    :							Optional[int]   			=       (
                              batch,
                              self.num_attention_heads,
                              past_key_values_length,
                              self._config.hidden_size // self.num_attention_heads,
                          )
                          _lowerCAmelCase    :							Tuple   			=       [
                              (torch.zeros(snake_case__   ), torch.zeros(snake_case__   )) for _ in range(self.num_layers   )
                          ]

            _lowerCAmelCase    :							Tuple   			=       common_inputs['attention_mask']
            if self.use_past:
                   _lowerCAmelCase    :							Any   			=       ordered_inputs['attention_mask'].dtype
                   _lowerCAmelCase    :							Union[str, Any]   			=       torch.cat(
                       [ordered_inputs['attention_mask'], torch.ones(snake_case__      ,	snake_case__      ,	dtype=snake_case__   )]      ,	dim=1   )

            return ordered_inputs







     @property
     def a	( self   ):

            '''simple docstring'''

            return 13

