





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_		:				List[Any]				=						logging.get_logger(__name__)

lowercase_		:				Union[str, Any]				=						{
    '''facebook/nllb-moe-54B''': '''https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json''',
}


class 			lowercase				(							a_     ):



						"""simple docstring"""

						_UpperCamelCase  :	List[str]								=	"nllb-moe"
						_UpperCamelCase  :	Union[str, Any]								=	["past_key_values"]
						_UpperCamelCase  :	Optional[int]								=	{"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}







						def __init__(    self			:		str ,				lowerCamelCase_			:		List[str]=12_81_12 ,				lowerCamelCase_			:		Any=10_24 ,				lowerCamelCase_			:		List[str]=12 ,				lowerCamelCase_			:		List[str]=40_96 ,				lowerCamelCase_			:		Optional[Any]=16 ,				lowerCamelCase_			:		Dict=12 ,				lowerCamelCase_			:		Tuple=40_96 ,				lowerCamelCase_			:		List[Any]=16 ,				lowerCamelCase_			:		List[str]=0.05 ,				lowerCamelCase_			:		Tuple=0.05 ,				lowerCamelCase_			:		Tuple=True ,				lowerCamelCase_			:		Tuple=True ,				lowerCamelCase_			:		Dict="relu" ,				lowerCamelCase_			:		Optional[Any]=10_24 ,				lowerCamelCase_			:		Optional[int]=0.1 ,				lowerCamelCase_			:		Optional[int]=0.1 ,				lowerCamelCase_			:		Dict=0.0 ,				lowerCamelCase_			:		List[Any]=0.02 ,				lowerCamelCase_			:		int=2 ,				lowerCamelCase_			:		List[str]=True ,				lowerCamelCase_			:		List[Any]=False ,				lowerCamelCase_			:		Dict="float32" ,				lowerCamelCase_			:		Union[str, Any]=False ,				lowerCamelCase_			:		Union[str, Any]=1_28 ,				lowerCamelCase_			:		Optional[Any]=64 ,				lowerCamelCase_			:		List[Any]=4 ,				lowerCamelCase_			:		int=4 ,				lowerCamelCase_			:		Dict=0.001 ,				lowerCamelCase_			:		Any=0.001 ,				lowerCamelCase_			:		str="all" ,				lowerCamelCase_			:		Optional[int]=False ,				lowerCamelCase_			:		List[Any]=False ,				lowerCamelCase_			:		Dict=1.0 ,				lowerCamelCase_			:		Optional[int]=0.2 ,				lowerCamelCase_			:		List[str]=1 ,				lowerCamelCase_			:		List[str]=0 ,				lowerCamelCase_			:		Tuple=2 ,				lowerCamelCase_			:		int=False ,				**lowerCamelCase_			:		Any ,				):





										'''simple docstring'''







										_snake_case :			List[str]  						=   vocab_size
										_snake_case :			int  						=   max_position_embeddings
										_snake_case :			List[Any]  						=   d_model
										_snake_case :			Any  						=   encoder_ffn_dim
										_snake_case :			List[Any]  						=   encoder_layers
										_snake_case :			Any  						=   encoder_attention_heads
										_snake_case :			List[Any]  						=   decoder_ffn_dim
										_snake_case :			List[Any]  						=   decoder_layers
										_snake_case :			List[str]  						=   decoder_attention_heads
										_snake_case :			List[Any]  						=   dropout
										_snake_case :			Union[str, Any]  						=   attention_dropout
										_snake_case :			Optional[Any]  						=   activation_dropout
										_snake_case :			int  						=   activation_function
										_snake_case :			List[Any]  						=   init_std
										_snake_case :			Union[str, Any]  						=   encoder_layerdrop
										_snake_case :			Tuple  						=   decoder_layerdrop
										_snake_case :			Optional[Any]  						=   use_cache
										_snake_case :			List[Any]  						=   encoder_layers
										_snake_case :			Dict  						=   scale_embedding  # scale factor will be sqrt(d_model) if True
										_snake_case :			Any  						=   router_z_loss_coef
										_snake_case :			Tuple  						=   router_aux_loss_coef
										_snake_case :			List[Any]  						=   decoder_sparse_step
										_snake_case :			Any  						=   encoder_sparse_step
										_snake_case :			Any  						=   num_experts
										_snake_case :			str  						=   expert_capacity
										_snake_case :			Any  						=   router_bias
										if router_dtype not in ["float32", "float16", "bfloat16"]:
														raise ValueError(f'''`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'''					)
										_snake_case :			Union[str, Any]  						=   router_dtype

										_snake_case :			int  						=   router_ignore_padding_tokens
										_snake_case :			Any  						=   batch_prioritized_routing
										_snake_case :			List[str]  						=   second_expert_policy
										_snake_case :			Optional[int]  						=   normalize_router_prob_before_dropping
										_snake_case :			int  						=   moe_eval_capacity_token_fraction
										_snake_case :			Tuple  						=   moe_token_dropout
										_snake_case :			Dict  						=   output_router_logits
										super().__init__(
										    pad_token_id=lowerCamelCase_ ,				bos_token_id=lowerCamelCase_ ,				eos_token_id=lowerCamelCase_ ,				is_encoder_decoder=lowerCamelCase_ ,				decoder_start_token_id=lowerCamelCase_ ,				**lowerCamelCase_ ,				)













import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin


class 			lowercase				(							a_		,			a_		,			a_     ):



						"""simple docstring"""

						@register_to_config
						def __init__(    self			:		Any ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		float ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		str ,				lowerCamelCase_			:		bool = False ,				):





										'''simple docstring'''







										super().__init__()

										_snake_case :			Optional[int]  						=   nn.Embedding(lowerCamelCase_ ,				lowerCamelCase_					)

										_snake_case :			List[str]  						=   nn.Embedding(lowerCamelCase_ ,				lowerCamelCase_					)
										_snake_case :			Union[str, Any]  						=   False

										_snake_case :			Tuple  						=   nn.Dropout(p=lowerCamelCase_					)

										_snake_case :			Union[str, Any]  						=   TaConfig(
										    vocab_size=lowerCamelCase_ ,				d_model=lowerCamelCase_ ,				num_heads=lowerCamelCase_ ,				d_kv=lowerCamelCase_ ,				d_ff=lowerCamelCase_ ,				dropout_rate=lowerCamelCase_ ,				feed_forward_proj=lowerCamelCase_ ,				is_decoder=lowerCamelCase_ ,				is_encoder_decoder=lowerCamelCase_ ,				)

										_snake_case :			Union[str, Any]  						=   nn.ModuleList()
										for lyr_num in range(lowerCamelCase_					):
														_snake_case :			Any  						=   TaBlock(lowerCamelCase_					)
														self.encoders.append(lowerCamelCase_					)

										_snake_case :			Tuple  						=   TaLayerNorm(lowerCamelCase_					)
										_snake_case :			List[str]  						=   nn.Dropout(p=lowerCamelCase_					)







						def 		__UpperCAmelCase       (    self			:		Optional[int] ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		Tuple					):





										'''simple docstring'''







										_snake_case :			Any  						=   self.token_embedder(lowerCamelCase_					)

										_snake_case :			List[Any]  						=   encoder_input_tokens.shape[1]
										_snake_case :			Any  						=   torch.arange(lowerCamelCase_ ,				device=encoder_input_tokens.device					)
										x += self.position_encoding(lowerCamelCase_					)

										_snake_case :			Tuple  						=   self.dropout_pre(lowerCamelCase_					)

										# inverted the attention mask
										_snake_case :			Dict  						=   encoder_input_tokens.size()
										_snake_case :			Optional[int]  						=   self.get_extended_attention_mask(lowerCamelCase_ ,				lowerCamelCase_					)

										for lyr in self.encoders:
														_snake_case :			str  						=   lyr(lowerCamelCase_ ,				lowerCamelCase_					)[0]
										_snake_case :			Any  						=   self.layer_norm(lowerCamelCase_					)

										return self.dropout_post(lowerCamelCase_					), encoder_inputs_mask






