
from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_						: Optional[Any]         =  logging.get_logger(__name__)

lowerCamelCase_						: Tuple         =  {
    """facebook/nllb-moe-54B""": """https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json""",
}

class        _UpperCamelCase				(					_A  ):






			'''simple docstring'''

			__UpperCamelCase      :						Union[str, Any]					      = """nllb-moe"""
			__UpperCamelCase      :						Optional[Any]					      = ["""past_key_values"""]
			__UpperCamelCase      :						Tuple					      = {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}

			def __init__(       self :       List[str]						,	snake_case_ :       int=12_8112						,	snake_case_ :       Optional[int]=1024						,	snake_case_ :       Union[str, Any]=12						,	snake_case_ :       Any=4096						,	snake_case_ :       Any=16						,	snake_case_ :       List[str]=12						,	snake_case_ :       int=4096						,	snake_case_ :       List[Any]=16						,	snake_case_ :       Tuple=0.05						,	snake_case_ :       Dict=0.05						,	snake_case_ :       Optional[int]=True						,	snake_case_ :       List[str]=True						,	snake_case_ :       Tuple="relu"						,	snake_case_ :       Optional[Any]=1024						,	snake_case_ :       List[str]=0.1						,	snake_case_ :       Dict=0.1						,	snake_case_ :       Any=0.0						,	snake_case_ :       List[str]=0.02						,	snake_case_ :       Tuple=2						,	snake_case_ :       Tuple=True						,	snake_case_ :       str=False						,	snake_case_ :       Any="float32"						,	snake_case_ :       int=False						,	snake_case_ :       Optional[Any]=128						,	snake_case_ :       List[str]=64						,	snake_case_ :       Union[str, Any]=4						,	snake_case_ :       List[str]=4						,	snake_case_ :       Any=0.001						,	snake_case_ :       List[str]=0.001						,	snake_case_ :       Optional[int]="all"						,	snake_case_ :       Tuple=False						,	snake_case_ :       Union[str, Any]=False						,	snake_case_ :       Tuple=1.0						,	snake_case_ :       List[Any]=0.2						,	snake_case_ :       Optional[Any]=1						,	snake_case_ :       Optional[Any]=0						,	snake_case_ :       List[Any]=2						,	snake_case_ :       Any=False						,	**snake_case_ :       Tuple						,	):
							UpperCamelCase_:							Any	=						vocab_size
							UpperCamelCase_:							int	=						max_position_embeddings
							UpperCamelCase_:							List[Any]	=						d_model
							UpperCamelCase_:							Tuple	=						encoder_ffn_dim
							UpperCamelCase_:							Tuple	=						encoder_layers
							UpperCamelCase_:							Union[str, Any]	=						encoder_attention_heads
							UpperCamelCase_:							Optional[int]	=						decoder_ffn_dim
							UpperCamelCase_:							Dict	=						decoder_layers
							UpperCamelCase_:							int	=						decoder_attention_heads
							UpperCamelCase_:							Any	=						dropout
							UpperCamelCase_:							Any	=						attention_dropout
							UpperCamelCase_:							List[Any]	=						activation_dropout
							UpperCamelCase_:							str	=						activation_function
							UpperCamelCase_:							str	=						init_std
							UpperCamelCase_:							Optional[int]	=						encoder_layerdrop
							UpperCamelCase_:							Optional[int]	=						decoder_layerdrop
							UpperCamelCase_:							Dict	=						use_cache
							UpperCamelCase_:							List[Any]	=						encoder_layers
							UpperCamelCase_:							List[str]	=						scale_embedding  # scale factor will be sqrt(d_model) if True
							UpperCamelCase_:							List[Any]	=						router_z_loss_coef
							UpperCamelCase_:							int	=						router_aux_loss_coef
							UpperCamelCase_:							Optional[Any]	=						decoder_sparse_step
							UpperCamelCase_:							int	=						encoder_sparse_step
							UpperCamelCase_:							Any	=						num_experts
							UpperCamelCase_:							Any	=						expert_capacity
							UpperCamelCase_:							int	=						router_bias
							if router_dtype not in ["float32", "float16", "bfloat16"]:
											raise ValueError(f'''`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'''      )
							UpperCamelCase_:							Tuple	=						router_dtype

							UpperCamelCase_:							int	=						router_ignore_padding_tokens
							UpperCamelCase_:							Optional[int]	=						batch_prioritized_routing
							UpperCamelCase_:							Tuple	=						second_expert_policy
							UpperCamelCase_:							Any	=						normalize_router_prob_before_dropping
							UpperCamelCase_:							List[Any]	=						moe_eval_capacity_token_fraction
							UpperCamelCase_:							Dict	=						moe_token_dropout
							UpperCamelCase_:							Tuple	=						output_router_logits
							super().__init__(
							    pad_token_id=snake_case_						,	bos_token_id=snake_case_						,	eos_token_id=snake_case_						,	is_encoder_decoder=snake_case_						,	decoder_start_token_id=snake_case_						,	**snake_case_						,	)








lowerCamelCase_						: Optional[Any]         =  """
# Transformers installation
! pip install transformers datasets
# To install from source instead of the last release, comment the command above and uncomment the following one.
# ! pip install git+https://github.com/huggingface/transformers.git
"""

lowerCamelCase_						: Union[str, Any]         =  [{"""type""": """code""", """content""": INSTALL_CONTENT}]
lowerCamelCase_						: Optional[Any]         =  {
    """{processor_class}""": """FakeProcessorClass""",
    """{model_class}""": """FakeModelClass""",
    """{object_class}""": """FakeObjectClass""",
}






