


'''simple docstring'''





import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_   )						->Optional[int]:
   assert isinstance(_lowerCamelCase  ,						_lowerCamelCase   )
   assert dataset.num_rows == 4
   assert dataset.num_columns == 3
   assert dataset.column_names == ["col_1", "col_2", "col_3"]
   for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("""keep_in_memory"""  ,						[False, True]   )
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )						->int:
   UpperCAmelCase							=     tmp_path / """cache"""
   UpperCAmelCase							=     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
      UpperCAmelCase							=     ParquetDatasetReader(_lowerCamelCase  ,						cache_dir=_lowerCamelCase  ,						keep_in_memory=_lowerCamelCase   ).read()
   _check_parquet_dataset(_lowerCamelCase  ,						_lowerCamelCase   )
@pytest.mark.parametrize(
    """features"""  ,						[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]  ,						)
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )						->int:
   UpperCAmelCase							=     tmp_path / """cache"""
   UpperCAmelCase							=     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   UpperCAmelCase							=     features.copy() if features else default_expected_features
   UpperCAmelCase							=     (
       Features({feature: Value(_lowerCamelCase   ) for feature, dtype in features.items()}   ) if features is not None else None
   )
   UpperCAmelCase							=     ParquetDatasetReader(_lowerCamelCase  ,						features=_lowerCamelCase  ,						cache_dir=_lowerCamelCase   ).read()
   _check_parquet_dataset(_lowerCamelCase  ,						_lowerCamelCase   )
@pytest.mark.parametrize("""split"""  ,						[None, NamedSplit("""train"""   ), """train""", """test"""]   )
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )						->Union[str, Any]:
   UpperCAmelCase							=     tmp_path / """cache"""
   UpperCAmelCase							=     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   UpperCAmelCase							=     ParquetDatasetReader(_lowerCamelCase  ,						cache_dir=_lowerCamelCase  ,						split=_lowerCamelCase   ).read()
   _check_parquet_dataset(_lowerCamelCase  ,						_lowerCamelCase   )
   assert dataset.split == split if split else "train"
@pytest.mark.parametrize("""path_type"""  ,						[str, list]   )
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )						->List[str]:
   if issubclass(_lowerCamelCase  ,						_lowerCamelCase   ):
      UpperCAmelCase							=     parquet_path
   elif issubclass(_lowerCamelCase  ,						_lowerCamelCase   ):
      UpperCAmelCase							=     [parquet_path]
   UpperCAmelCase							=     tmp_path / """cache"""
   UpperCAmelCase							=     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   UpperCAmelCase							=     ParquetDatasetReader(_lowerCamelCase  ,						cache_dir=_lowerCamelCase   ).read()
   _check_parquet_dataset(_lowerCamelCase  ,						_lowerCamelCase   )
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_=("train",)   )						->Optional[int]:
   assert isinstance(_lowerCamelCase  ,						_lowerCamelCase   )
   for split in splits:
      UpperCAmelCase							=     dataset_dict[split]
      assert dataset.num_rows == 4
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_1", "col_2", "col_3"]
      for feature, expected_dtype in expected_features.items():
         assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("""keep_in_memory"""  ,						[False, True]   )
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )						->Union[str, Any]:
   UpperCAmelCase							=     tmp_path / """cache"""
   UpperCAmelCase							=     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
      UpperCAmelCase							=     ParquetDatasetReader(
          {"""train""": parquet_path}  ,						cache_dir=_lowerCamelCase  ,						keep_in_memory=_lowerCamelCase   ).read()
   _check_parquet_datasetdict(_lowerCamelCase  ,						_lowerCamelCase   )
@pytest.mark.parametrize(
    """features"""  ,						[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]  ,						)
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )						->Optional[Any]:
   UpperCAmelCase							=     tmp_path / """cache"""
   UpperCAmelCase							=     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   UpperCAmelCase							=     features.copy() if features else default_expected_features
   UpperCAmelCase							=     (
       Features({feature: Value(_lowerCamelCase   ) for feature, dtype in features.items()}   ) if features is not None else None
   )
   UpperCAmelCase							=     ParquetDatasetReader({"""train""": parquet_path}  ,						features=_lowerCamelCase  ,						cache_dir=_lowerCamelCase   ).read()
   _check_parquet_datasetdict(_lowerCamelCase  ,						_lowerCamelCase   )
@pytest.mark.parametrize("""split"""  ,						[None, NamedSplit("""train"""   ), """train""", """test"""]   )
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )						->Any:
   if split:
      UpperCAmelCase							=     {split: parquet_path}
   else:
      UpperCAmelCase							=     """train"""
      UpperCAmelCase							=     {"""train""": parquet_path, """test""": parquet_path}
   UpperCAmelCase							=     tmp_path / """cache"""
   UpperCAmelCase							=     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   UpperCAmelCase							=     ParquetDatasetReader(_lowerCamelCase  ,						cache_dir=_lowerCamelCase   ).read()
   _check_parquet_datasetdict(_lowerCamelCase  ,						_lowerCamelCase  ,						splits=list(path.keys()   )   )
   assert all(dataset[split].split == split for split in path.keys()   )
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_   )						->List[Any]:
   UpperCAmelCase							=     ParquetDatasetWriter(_lowerCamelCase  ,						tmp_path / """foo.parquet"""   )
   assert writer.write() > 0
   UpperCAmelCase							=     pq.ParquetFile(tmp_path / """foo.parquet"""   )
   UpperCAmelCase							=     pf.read()
   assert dataset.data.table == output_table
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_   )						->Any:
   UpperCAmelCase							=     str(shared_datadir / """test_image_rgb.jpg"""   )
   UpperCAmelCase							=     {"""image""": [image_path]}
   UpperCAmelCase							=     Features({"""image""": Image()}   )
   UpperCAmelCase							=     Dataset.from_dict(_lowerCamelCase  ,						features=_lowerCamelCase   )
   UpperCAmelCase							=     ParquetDatasetWriter(_lowerCamelCase  ,						tmp_path / """foo.parquet"""   )
   assert writer.write() > 0

   UpperCAmelCase							=     Dataset.from_parquet(str(tmp_path / """foo.parquet"""   )   )
   assert dataset.features == reloaded_dataset.features

   UpperCAmelCase							=     ParquetDatasetReader(str(tmp_path / """foo.parquet"""   )  ,						streaming=_lowerCamelCase   ).read()
   assert dataset.features == reloaded_iterable_dataset.features







@pytest.mark.parametrize(
    """feature, expected"""  ,						[
        (Features({"""foo""": Value("""int32"""   )}   ), None),
        (Features({"""image""": Image(), """foo""": Value("""int32"""   )}   ), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({"""nested""": Sequence(Audio()   )}   ), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]  ,						)
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_   )						->int:
   assert get_writer_batch_size(_lowerCamelCase   ) == expected







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__a     =			logging.get_logger(__name__)

__a     =			{
    """facebook/nllb-moe-54B""": """https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json""",
}






class   __lowercase				(  __snake_case  ):
      UpperCamelCase								=       '''nllb-moe'''
      UpperCamelCase								=       ['''past_key_values''']
      UpperCamelCase								=       {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}





      def __init__(		self						:					Optional[int]					,       __lowerCamelCase						:					Optional[Any]=1_2_8_1_1_2					,       __lowerCamelCase						:					Dict=1_0_2_4					,       __lowerCamelCase						:					Optional[int]=1_2					,       __lowerCamelCase						:					Union[str, Any]=4_0_9_6					,       __lowerCamelCase						:					List[str]=1_6					,       __lowerCamelCase						:					List[str]=1_2					,       __lowerCamelCase						:					int=4_0_9_6					,       __lowerCamelCase						:					Tuple=1_6					,       __lowerCamelCase						:					str=0.05					,       __lowerCamelCase						:					List[str]=0.05					,       __lowerCamelCase						:					List[Any]=True					,       __lowerCamelCase						:					Tuple=True					,       __lowerCamelCase						:					str="relu"					,       __lowerCamelCase						:					Dict=1_0_2_4					,       __lowerCamelCase						:					List[str]=0.1					,       __lowerCamelCase						:					Optional[int]=0.1					,       __lowerCamelCase						:					List[Any]=0.0					,       __lowerCamelCase						:					Optional[Any]=0.02					,       __lowerCamelCase						:					Dict=2					,       __lowerCamelCase						:					Union[str, Any]=True					,       __lowerCamelCase						:					Any=False					,       __lowerCamelCase						:					Tuple="float32"					,       __lowerCamelCase						:					Any=False					,       __lowerCamelCase						:					Optional[int]=1_2_8					,       __lowerCamelCase						:					List[str]=6_4					,       __lowerCamelCase						:					List[Any]=4					,       __lowerCamelCase						:					Tuple=4					,       __lowerCamelCase						:					str=0.001					,       __lowerCamelCase						:					Optional[int]=0.001					,       __lowerCamelCase						:					Tuple="all"					,       __lowerCamelCase						:					Any=False					,       __lowerCamelCase						:					Optional[int]=False					,       __lowerCamelCase						:					List[str]=1.0					,       __lowerCamelCase						:					Dict=0.2					,       __lowerCamelCase						:					Union[str, Any]=1					,       __lowerCamelCase						:					int=0					,       __lowerCamelCase						:					Dict=2					,       __lowerCamelCase						:					int=False					,       **__lowerCamelCase						:					str					,       )    ->   int:






         """simple docstring"""







         UpperCAmelCase							=     vocab_size
         UpperCAmelCase							=     max_position_embeddings
         UpperCAmelCase							=     d_model
         UpperCAmelCase							=     encoder_ffn_dim
         UpperCAmelCase							=     encoder_layers
         UpperCAmelCase							=     encoder_attention_heads
         UpperCAmelCase							=     decoder_ffn_dim
         UpperCAmelCase							=     decoder_layers
         UpperCAmelCase							=     decoder_attention_heads
         UpperCAmelCase							=     dropout
         UpperCAmelCase							=     attention_dropout
         UpperCAmelCase							=     activation_dropout
         UpperCAmelCase							=     activation_function
         UpperCAmelCase							=     init_std
         UpperCAmelCase							=     encoder_layerdrop
         UpperCAmelCase							=     decoder_layerdrop
         UpperCAmelCase							=     use_cache
         UpperCAmelCase							=     encoder_layers
         UpperCAmelCase							=     scale_embedding  # scale factor will be sqrt(d_model) if True
         UpperCAmelCase							=     router_z_loss_coef
         UpperCAmelCase							=     router_aux_loss_coef
         UpperCAmelCase							=     decoder_sparse_step
         UpperCAmelCase							=     encoder_sparse_step
         UpperCAmelCase							=     num_experts
         UpperCAmelCase							=     expert_capacity
         UpperCAmelCase							=     router_bias
         if router_dtype not in ["float32", "float16", "bfloat16"]:
            raise ValueError(F"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""       )
         UpperCAmelCase							=     router_dtype

         UpperCAmelCase							=     router_ignore_padding_tokens
         UpperCAmelCase							=     batch_prioritized_routing
         UpperCAmelCase							=     second_expert_policy
         UpperCAmelCase							=     normalize_router_prob_before_dropping
         UpperCAmelCase							=     moe_eval_capacity_token_fraction
         UpperCAmelCase							=     moe_token_dropout
         UpperCAmelCase							=     output_router_logits
         super().__init__(
             pad_token_id=__lowerCamelCase					,       bos_token_id=__lowerCamelCase					,       eos_token_id=__lowerCamelCase					,       is_encoder_decoder=__lowerCamelCase					,       decoder_start_token_id=__lowerCamelCase					,       **__lowerCamelCase					,       )



