



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case	=						logging.get_logger(__name__)

_snake_case	=						{
    "google/switch-base-8": "https://huggingface.co/google/switch-base-8/blob/main/config.json",
}



class 						UpperCAmelCase_					( a):
							lowerCamelCase__         =	'switch_transformers'
							lowerCamelCase__         =	['past_key_values']
							lowerCamelCase__         =	{'hidden_size': 'd_model', 'num_attention_heads': 'num_heads', 'num_hidden_layers': 'num_layers'}







							def __init__(     self,    __a=3_2128,    __a=768,    __a=64,    __a=2048,    __a=64,    __a=12,    __a=3,    __a=12,    __a=3,    __a=12,    __a=8,    __a=False,    __a=0.01,    __a="float32",    __a=False,    __a=32,    __a=128,    __a=0.1,    __a=1E-6,    __a=0.001,    __a=0.001,    __a=1.0,    __a="relu",    __a=True,    __a=False,    __a=True,    __a=0,    __a=1,    **__a,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							str				     =					vocab_size
														_lowerCAmelCase  :							str				     =					d_model
														_lowerCAmelCase  :							Optional[Any]				     =					d_kv
														_lowerCAmelCase  :							Dict				     =					d_ff

														_lowerCAmelCase  :							List[str]				     =					num_sparse_encoder_layers

														_lowerCAmelCase  :							Tuple				     =					num_layers
														_lowerCAmelCase  :							int				     =					(
														    num_decoder_layers if num_decoder_layers is not None else self.num_layers
														)  # default = symmetry
														_lowerCAmelCase  :							List[Any]				     =					num_sparse_decoder_layers

														# This tells us, each how many encoder layer we'll have to set a sparse layer.
														if self.num_sparse_encoder_layers > 0:
																					_lowerCAmelCase  :							Any				     =					self.num_layers // self.num_sparse_encoder_layers
														else:
																					_lowerCAmelCase  :							Optional[Any]				     =					self.num_layers  # HACK: this will create 0 sparse layers

														# This tells us, each how many encoder layer we'll have to set a sparse layer.
														if self.num_sparse_decoder_layers > 0:
																					_lowerCAmelCase  :							Optional[Any]				     =					self.num_decoder_layers // self.num_sparse_decoder_layers
														else:
																					_lowerCAmelCase  :							List[str]				     =					self.num_decoder_layers  # HACK: this will create 0 sparse layers

														_lowerCAmelCase  :							Any				     =					num_heads
														_lowerCAmelCase  :							int				     =					num_experts
														_lowerCAmelCase  :							str				     =					expert_capacity
														_lowerCAmelCase  :							int				     =					router_bias
														_lowerCAmelCase  :							Optional[int]				     =					router_jitter_noise
														if router_dtype not in ["float32", "float16", "bfloat16"]:
																					raise ValueError(f"`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}")
														_lowerCAmelCase  :							str				     =					router_dtype

														_lowerCAmelCase  :							Tuple				     =					router_ignore_padding_tokens
														_lowerCAmelCase  :							Dict				     =					relative_attention_num_buckets
														_lowerCAmelCase  :							List[Any]				     =					relative_attention_max_distance

														_lowerCAmelCase  :							Any				     =					dropout_rate
														_lowerCAmelCase  :							str				     =					layer_norm_epsilon
														_lowerCAmelCase  :							List[str]				     =					initializer_factor
														_lowerCAmelCase  :							Any				     =					feed_forward_proj
														_lowerCAmelCase  :							str				     =					use_cache
														_lowerCAmelCase  :							str				     =					add_router_probs

														_lowerCAmelCase  :							List[Any]				     =					router_z_loss_coef
														_lowerCAmelCase  :							Optional[int]				     =					router_aux_loss_coef

														_lowerCAmelCase  :							Optional[Any]				     =					self.feed_forward_proj.split("-")
														_lowerCAmelCase  :							Any				     =					act_info[-1]
														_lowerCAmelCase  :							List[str]				     =					act_info[0] == "gated"

														if len(__a) > 1 and act_info[0] != "gated" or len(__a) > 2:
																					raise ValueError(
																					    f"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."
																					    "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
																					    "'gated-gelu' or 'relu'")

														# for backwards compatibility
														if feed_forward_proj == "gated-gelu":
																					_lowerCAmelCase  :							Optional[Any]				     =					"gelu_new"

														super().__init__(
														    pad_token_id=__a,    eos_token_id=__a,    is_encoder_decoder=__a,    **__a,    )





from __future__ import annotations






def        A       (      _lowerCamelCase						):





							'''simple docstring'''




							if not nums:
														raise ValueError("List is empty"						)
							return sum(_lowerCamelCase						) / len(_lowerCamelCase						)


if __name__ == "__main__":
						import doctest

						doctest.testmod()
