


import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.text import TextDatasetReader

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
def      snake_case_						( __lowercase			,       __lowercase       ):
   assert isinstance(__lowercase			,       __lowercase       )
   assert dataset.num_rows == 4
   assert dataset.num_columns == 1
   assert dataset.column_names == ["text"]
   for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize('''keep_in_memory'''			,       [False, True]       )
def      snake_case_						( __lowercase			,       __lowercase			,       __lowercase       ):
   UpperCAmelCase_							:      Any						=   tmp_path / "cache"
   UpperCAmelCase_							:      Any						=   {"text": "string"}
   with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
      UpperCAmelCase_							:      Union[str, Any]						=   TextDatasetReader(__lowercase			,       cache_dir=__lowercase			,       keep_in_memory=__lowercase       ).read()
   _check_text_dataset(__lowercase			,       __lowercase       )
@pytest.mark.parametrize(
    '''features'''			,       [
        None,
        {'''text''': '''string'''},
        {'''text''': '''int32'''},
        {'''text''': '''float32'''},
    ]			,       )
def      snake_case_						( __lowercase			,       __lowercase			,       __lowercase       ):
   UpperCAmelCase_							:      int						=   tmp_path / "cache"
   UpperCAmelCase_							:      Optional[Any]						=   {"text": "string"}
   UpperCAmelCase_							:      int						=   features.copy() if features else default_expected_features
   UpperCAmelCase_							:      List[str]						=   (
       Features({feature: Value(__lowercase       ) for feature, dtype in features.items()}       ) if features is not None else None
   )
   UpperCAmelCase_							:      str						=   TextDatasetReader(__lowercase			,       features=__lowercase			,       cache_dir=__lowercase       ).read()
   _check_text_dataset(__lowercase			,       __lowercase       )
@pytest.mark.parametrize('''split'''			,       [None, NamedSplit('''train'''       ), '''train''', '''test''']       )
def      snake_case_						( __lowercase			,       __lowercase			,       __lowercase       ):
   UpperCAmelCase_							:      Optional[Any]						=   tmp_path / "cache"
   UpperCAmelCase_							:      Tuple						=   {"text": "string"}
   UpperCAmelCase_							:      List[Any]						=   TextDatasetReader(__lowercase			,       cache_dir=__lowercase			,       split=__lowercase       ).read()
   _check_text_dataset(__lowercase			,       __lowercase       )
   assert dataset.split == split if split else "train"
@pytest.mark.parametrize('''path_type'''			,       [str, list]       )
def      snake_case_						( __lowercase			,       __lowercase			,       __lowercase       ):



   if issubclass(__lowercase			,       __lowercase       ):
      UpperCAmelCase_							:      Tuple						=   text_path
   elif issubclass(__lowercase			,       __lowercase       ):
      UpperCAmelCase_							:      Union[str, Any]						=   [text_path]
   UpperCAmelCase_							:      List[Any]						=   tmp_path / "cache"
   UpperCAmelCase_							:      str						=   {"text": "string"}
   UpperCAmelCase_							:      Dict						=   TextDatasetReader(__lowercase			,       cache_dir=__lowercase       ).read()
   _check_text_dataset(__lowercase			,       __lowercase       )
def      snake_case_						( __lowercase			,       __lowercase			,       __lowercase=("train",)       ):
   assert isinstance(__lowercase			,       __lowercase       )
   for split in splits:
      UpperCAmelCase_							:      List[Any]						=   dataset_dict[split]
      assert dataset.num_rows == 4
      assert dataset.num_columns == 1
      assert dataset.column_names == ["text"]
      for feature, expected_dtype in expected_features.items():
         assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize('''keep_in_memory'''			,       [False, True]       )
def      snake_case_						( __lowercase			,       __lowercase			,       __lowercase       ):
   UpperCAmelCase_							:      Optional[int]						=   tmp_path / "cache"
   UpperCAmelCase_							:      Union[str, Any]						=   {"text": "string"}
   with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
      UpperCAmelCase_							:      Tuple						=   TextDatasetReader({'''train''': text_path}			,       cache_dir=__lowercase			,       keep_in_memory=__lowercase       ).read()
   _check_text_datasetdict(__lowercase			,       __lowercase       )
@pytest.mark.parametrize(
    '''features'''			,       [
        None,
        {'''text''': '''string'''},
        {'''text''': '''int32'''},
        {'''text''': '''float32'''},
    ]			,       )
def      snake_case_						( __lowercase			,       __lowercase			,       __lowercase       ):
   UpperCAmelCase_							:      Optional[int]						=   tmp_path / "cache"
   # CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
   UpperCAmelCase_							:      Optional[int]						=   {"text": "string"}
   UpperCAmelCase_							:      List[str]						=   features.copy() if features else default_expected_features
   UpperCAmelCase_							:      str						=   (
       Features({feature: Value(__lowercase       ) for feature, dtype in features.items()}       ) if features is not None else None
   )
   UpperCAmelCase_							:      Union[str, Any]						=   TextDatasetReader({'''train''': text_path}			,       features=__lowercase			,       cache_dir=__lowercase       ).read()
   _check_text_datasetdict(__lowercase			,       __lowercase       )


@pytest.mark.parametrize('''split'''			,       [None, NamedSplit('''train'''       ), '''train''', '''test''']       )
def      snake_case_						( __lowercase			,       __lowercase			,       __lowercase       ):



   if split:
      UpperCAmelCase_							:      int						=   {split: text_path}
   else:
      UpperCAmelCase_							:      str						=   "train"
      UpperCAmelCase_							:      int						=   {"train": text_path, "test": text_path}
   UpperCAmelCase_							:      Any						=   tmp_path / "cache"
   UpperCAmelCase_							:      Optional[Any]						=   {"text": "string"}
   UpperCAmelCase_							:      List[Any]						=   TextDatasetReader(__lowercase			,       cache_dir=__lowercase       ).read()
   _check_text_datasetdict(__lowercase			,       __lowercase			,       splits=list(path.keys()       )       )
   assert all(dataset[split].split == split for split in path.keys()       )






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase				:  Any      					=							logging.get_logger(__name__)

__UpperCamelCase				:  Dict      					=							{
    'bigcode/gpt_bigcode-santacoder': 'https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json',
}





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     List[str]			= 'gpt_bigcode'
    A_  :     Optional[Any]			= ['past_key_values']
    A_  :     Optional[int]			= {
        'hidden_size': 'n_embd',
        'max_position_embeddings': 'n_positions',
        'num_attention_heads': 'n_head',
        'num_hidden_layers': 'n_layer',
    }





    def __init__(	self				:   int       ,   __snake_case				:   Dict=50_257       ,   __snake_case				:   List[str]=1_024       ,   __snake_case				:   Dict=768       ,   __snake_case				:   Optional[int]=12       ,   __snake_case				:   str=12       ,   __snake_case				:   List[str]=None       ,   __snake_case				:   List[str]="gelu_pytorch_tanh"       ,   __snake_case				:   Optional[Any]=0.1       ,   __snake_case				:   str=0.1       ,   __snake_case				:   Optional[Any]=0.1       ,   __snake_case				:   List[str]=1E-5       ,   __snake_case				:   Dict=0.02       ,   __snake_case				:   Optional[int]=True       ,   __snake_case				:   Tuple=True       ,   __snake_case				:   Optional[Any]=50_256       ,   __snake_case				:   Optional[Any]=50_256       ,   __snake_case				:   Optional[Any]=True       ,   __snake_case				:   Optional[Any]=True       ,   __snake_case				:   List[Any]=True       ,   **__snake_case				:   List[str]       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Any						=   vocab_size
       UpperCAmelCase_							:      int						=   n_positions
       UpperCAmelCase_							:      Any						=   n_embd
       UpperCAmelCase_							:      Union[str, Any]						=   n_layer
       UpperCAmelCase_							:      List[str]						=   n_head
       UpperCAmelCase_							:      List[Any]						=   n_inner
       UpperCAmelCase_							:      Optional[int]						=   activation_function
       UpperCAmelCase_							:      str						=   resid_pdrop
       UpperCAmelCase_							:      Optional[Any]						=   embd_pdrop
       UpperCAmelCase_							:      str						=   attn_pdrop
       UpperCAmelCase_							:      Any						=   layer_norm_epsilon
       UpperCAmelCase_							:      List[str]						=   initializer_range
       UpperCAmelCase_							:      Optional[int]						=   scale_attn_weights
       UpperCAmelCase_							:      Union[str, Any]						=   use_cache
       UpperCAmelCase_							:      Dict						=   attention_softmax_in_fpaa
       UpperCAmelCase_							:      Union[str, Any]						=   scale_attention_softmax_in_fpaa
       UpperCAmelCase_							:      Optional[int]						=   multi_query

       UpperCAmelCase_							:      Optional[Any]						=   bos_token_id
       UpperCAmelCase_							:      Tuple						=   eos_token_id

       super().__init__(bos_token_id=__snake_case       ,   eos_token_id=__snake_case       ,   **__snake_case	)