



'''simple docstring'''
import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases





def  _a(  UpperCamelCase__				:						Dict,	UpperCamelCase__				:						Optional[int]							):




      '''simple docstring'''






      assert isinstance(UpperCamelCase__,	UpperCamelCase__							)
      assert dataset.num_rows == 4
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_1", "col_2", "col_3"]
      for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('''keep_in_memory''',	[False, True]							)
def  _a(  UpperCamelCase__				:						str,	UpperCamelCase__				:						List[Any],	UpperCamelCase__				:						Optional[int]							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=tmp_path / '''cache'''
      SCREAMING_SNAKE_CASE__     :				Tuple									={'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
            SCREAMING_SNAKE_CASE__     :				Optional[Any]									=ParquetDatasetReader(UpperCamelCase__,	cache_dir=UpperCamelCase__,	keep_in_memory=UpperCamelCase__							).read()
      _check_parquet_dataset(UpperCamelCase__,	UpperCamelCase__							)





@pytest.mark.parametrize(
    '''features''',	[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ],	)
def  _a(  UpperCamelCase__				:						List[Any],	UpperCamelCase__				:						Union[str, Any],	UpperCamelCase__				:						Optional[int]							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__     :				int									=tmp_path / '''cache'''
      SCREAMING_SNAKE_CASE__     :				Dict									={'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      SCREAMING_SNAKE_CASE__     :				Dict									=features.copy() if features else default_expected_features
      SCREAMING_SNAKE_CASE__     :				str									=(
          Features({feature: Value(UpperCamelCase__							) for feature, dtype in features.items()}							) if features is not None else None
      )
      SCREAMING_SNAKE_CASE__     :				Any									=ParquetDatasetReader(UpperCamelCase__,	features=UpperCamelCase__,	cache_dir=UpperCamelCase__							).read()
      _check_parquet_dataset(UpperCamelCase__,	UpperCamelCase__							)





@pytest.mark.parametrize('''split''',	[None, NamedSplit('''train'''							), '''train''', '''test''']							)
def  _a(  UpperCamelCase__				:						str,	UpperCamelCase__				:						List[str],	UpperCamelCase__				:						List[Any]							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__     :				Optional[Any]									=tmp_path / '''cache'''
      SCREAMING_SNAKE_CASE__     :				Any									={'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      SCREAMING_SNAKE_CASE__     :				int									=ParquetDatasetReader(UpperCamelCase__,	cache_dir=UpperCamelCase__,	split=UpperCamelCase__							).read()
      _check_parquet_dataset(UpperCamelCase__,	UpperCamelCase__							)
      assert dataset.split == split if split else "train"





@pytest.mark.parametrize('''path_type''',	[str, list]							)
def  _a(  UpperCamelCase__				:						Any,	UpperCamelCase__				:						Dict,	UpperCamelCase__				:						List[Any]							):




      '''simple docstring'''






      if issubclass(UpperCamelCase__,	UpperCamelCase__							):
            SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=parquet_path
      elif issubclass(UpperCamelCase__,	UpperCamelCase__							):
            SCREAMING_SNAKE_CASE__     :				Tuple									=[parquet_path]
      SCREAMING_SNAKE_CASE__     :				Optional[int]									=tmp_path / '''cache'''
      SCREAMING_SNAKE_CASE__     :				Optional[Any]									={'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      SCREAMING_SNAKE_CASE__     :				Optional[int]									=ParquetDatasetReader(UpperCamelCase__,	cache_dir=UpperCamelCase__							).read()
      _check_parquet_dataset(UpperCamelCase__,	UpperCamelCase__							)





def  _a(  UpperCamelCase__				:						Tuple,	UpperCamelCase__				:						Any,	UpperCamelCase__				:						List[str]=("train",)							):




      '''simple docstring'''






      assert isinstance(UpperCamelCase__,	UpperCamelCase__							)
      for split in splits:
            SCREAMING_SNAKE_CASE__     :				Tuple									=dataset_dict[split]
            assert dataset.num_rows == 4
            assert dataset.num_columns == 3
            assert dataset.column_names == ["col_1", "col_2", "col_3"]
            for feature, expected_dtype in expected_features.items():
                  assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('''keep_in_memory''',	[False, True]							)
def  _a(  UpperCamelCase__				:						Optional[Any],	UpperCamelCase__				:						str,	UpperCamelCase__				:						Dict							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__     :				Tuple									=tmp_path / '''cache'''
      SCREAMING_SNAKE_CASE__     :				int									={'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
            SCREAMING_SNAKE_CASE__     :				List[Any]									=ParquetDatasetReader(
                {'''train''': parquet_path},	cache_dir=UpperCamelCase__,	keep_in_memory=UpperCamelCase__							).read()
      _check_parquet_datasetdict(UpperCamelCase__,	UpperCamelCase__							)





@pytest.mark.parametrize(
    '''features''',	[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ],	)
def  _a(  UpperCamelCase__				:						List[Any],	UpperCamelCase__				:						int,	UpperCamelCase__				:						List[str]							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__     :				Tuple									=tmp_path / '''cache'''
      SCREAMING_SNAKE_CASE__     :				List[str]									={'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      SCREAMING_SNAKE_CASE__     :				Dict									=features.copy() if features else default_expected_features
      SCREAMING_SNAKE_CASE__     :				List[Any]									=(
          Features({feature: Value(UpperCamelCase__							) for feature, dtype in features.items()}							) if features is not None else None
      )
      SCREAMING_SNAKE_CASE__     :				List[Any]									=ParquetDatasetReader({'''train''': parquet_path},	features=UpperCamelCase__,	cache_dir=UpperCamelCase__							).read()
      _check_parquet_datasetdict(UpperCamelCase__,	UpperCamelCase__							)





@pytest.mark.parametrize('''split''',	[None, NamedSplit('''train'''							), '''train''', '''test''']							)
def  _a(  UpperCamelCase__				:						Optional[Any],	UpperCamelCase__				:						Any,	UpperCamelCase__				:						Optional[int]							):




      '''simple docstring'''






      if split:
            SCREAMING_SNAKE_CASE__     :				Optional[Any]									={split: parquet_path}
      else:
            SCREAMING_SNAKE_CASE__     :				Optional[Any]									='''train'''
            SCREAMING_SNAKE_CASE__     :				str									={'''train''': parquet_path, '''test''': parquet_path}
      SCREAMING_SNAKE_CASE__     :				Any									=tmp_path / '''cache'''
      SCREAMING_SNAKE_CASE__     :				Optional[int]									={'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      SCREAMING_SNAKE_CASE__     :				Optional[Any]									=ParquetDatasetReader(UpperCamelCase__,	cache_dir=UpperCamelCase__							).read()
      _check_parquet_datasetdict(UpperCamelCase__,	UpperCamelCase__,	splits=list(path.keys()							)							)
      assert all(dataset[split].split == split for split in path.keys()							)





def  _a(  UpperCamelCase__				:						Dict,	UpperCamelCase__				:						Union[str, Any]							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__     :				List[str]									=ParquetDatasetWriter(UpperCamelCase__,	tmp_path / '''foo.parquet'''							)
      assert writer.write() > 0
      SCREAMING_SNAKE_CASE__     :				List[Any]									=pq.ParquetFile(tmp_path / '''foo.parquet'''							)
      SCREAMING_SNAKE_CASE__     :				Any									=pf.read()
      assert dataset.data.table == output_table





def  _a(  UpperCamelCase__				:						Tuple,	UpperCamelCase__				:						str							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__     :				str									=str(shared_datadir / '''test_image_rgb.jpg'''							)
      SCREAMING_SNAKE_CASE__     :				List[Any]									={'''image''': [image_path]}
      SCREAMING_SNAKE_CASE__     :				Optional[Any]									=Features({'''image''': Image()}							)
      SCREAMING_SNAKE_CASE__     :				List[Any]									=Dataset.from_dict(UpperCamelCase__,	features=UpperCamelCase__							)
      SCREAMING_SNAKE_CASE__     :				List[Any]									=ParquetDatasetWriter(UpperCamelCase__,	tmp_path / '''foo.parquet'''							)
      assert writer.write() > 0

      SCREAMING_SNAKE_CASE__     :				List[Any]									=Dataset.from_parquet(str(tmp_path / '''foo.parquet'''							)							)
      assert dataset.features == reloaded_dataset.features

      SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=ParquetDatasetReader(str(tmp_path / '''foo.parquet'''							),	streaming=UpperCamelCase__							).read()
      assert dataset.features == reloaded_iterable_dataset.features







@pytest.mark.parametrize(
    '''feature, expected''',	[
        (Features({'''foo''': Value('''int32'''							)}							), None),
        (Features({'''image''': Image(), '''foo''': Value('''int32'''							)}							), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({'''nested''': Sequence(Audio()							)}							), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ],	)
def  _a(  UpperCamelCase__				:						List[Any],	UpperCamelCase__				:						Tuple							):




      '''simple docstring'''






      assert get_writer_batch_size(UpperCamelCase__							) == expected







'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_					= logging.get_logger(__name__)

a_					= {
    'bigcode/gpt_bigcode-santacoder': 'https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json',
}






class      __SCREAMING_SNAKE_CASE      (			lowerCamelCase				):
 snake_case_  							=							"""gpt_bigcode"""
 snake_case_  							=							["""past_key_values"""]
 snake_case_  							=							{
     """hidden_size""": """n_embd""",
     """max_position_embeddings""": """n_positions""",
     """num_attention_heads""": """n_head""",
     """num_hidden_layers""": """n_layer""",
 }




 def __init__(      self	:							Any					,      __lowercase	:							Any=5_02_57					,      __lowercase	:							int=10_24					,      __lowercase	:							List[str]=7_68					,      __lowercase	:							Optional[int]=12					,      __lowercase	:							Dict=12					,      __lowercase	:							List[str]=None					,      __lowercase	:							int="gelu_pytorch_tanh"					,      __lowercase	:							Union[str, Any]=0.1					,      __lowercase	:							Optional[int]=0.1					,      __lowercase	:							Optional[int]=0.1					,      __lowercase	:							Optional[Any]=1e-5					,      __lowercase	:							List[str]=0.02					,      __lowercase	:							Tuple=True					,      __lowercase	:							Optional[Any]=True					,      __lowercase	:							Union[str, Any]=5_02_56					,      __lowercase	:							List[Any]=5_02_56					,      __lowercase	:							Union[str, Any]=True					,      __lowercase	:							List[str]=True					,      __lowercase	:							Dict=True					,      **__lowercase	:							List[Any]					,      )	->						Optional[int]:
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=vocab_size
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=n_positions
       SCREAMING_SNAKE_CASE__     :				Dict									=n_embd
       SCREAMING_SNAKE_CASE__     :				Dict									=n_layer
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=n_head
       SCREAMING_SNAKE_CASE__     :				List[str]									=n_inner
       SCREAMING_SNAKE_CASE__     :				List[str]									=activation_function
       SCREAMING_SNAKE_CASE__     :				List[Any]									=resid_pdrop
       SCREAMING_SNAKE_CASE__     :				List[Any]									=embd_pdrop
       SCREAMING_SNAKE_CASE__     :				List[str]									=attn_pdrop
       SCREAMING_SNAKE_CASE__     :				Dict									=layer_norm_epsilon
       SCREAMING_SNAKE_CASE__     :				List[str]									=initializer_range
       SCREAMING_SNAKE_CASE__     :				List[Any]									=scale_attn_weights
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=use_cache
       SCREAMING_SNAKE_CASE__     :				Dict									=attention_softmax_in_fpaa
       SCREAMING_SNAKE_CASE__     :				int									=scale_attention_softmax_in_fpaa
       SCREAMING_SNAKE_CASE__     :				Dict									=multi_query

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=bos_token_id
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=eos_token_id

       super().__init__(bos_token_id=__lowercase					,      eos_token_id=__lowercase					,      **__lowercase      )