


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_UpperCAmelCase			:  Optional[int]     ={
    """configuration_megatron_bert""": ["""MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MegatronBertConfig"""],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_UpperCAmelCase			:  Dict     =[
						    """MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """MegatronBertForCausalLM""",
						    """MegatronBertForMaskedLM""",
						    """MegatronBertForMultipleChoice""",
						    """MegatronBertForNextSentencePrediction""",
						    """MegatronBertForPreTraining""",
						    """MegatronBertForQuestionAnswering""",
						    """MegatronBertForSequenceClassification""",
						    """MegatronBertForTokenClassification""",
						    """MegatronBertModel""",
						    """MegatronBertPreTrainedModel""",
						]

if TYPE_CHECKING:
						from .configuration_megatron_bert import MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MegatronBertConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_megatron_bert import (
												    MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    MegatronBertForCausalLM,
												    MegatronBertForMaskedLM,
												    MegatronBertForMultipleChoice,
												    MegatronBertForNextSentencePrediction,
												    MegatronBertForPreTraining,
												    MegatronBertForQuestionAnswering,
												    MegatronBertForSequenceClassification,
												    MegatronBertForTokenClassification,
												    MegatronBertModel,
												    MegatronBertPreTrainedModel,
												)

else:
						import sys

						_UpperCAmelCase			:  str     =_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCAmelCase			:  Optional[int]     =logging.get_logger(__name__)

_UpperCAmelCase			:  Union[str, Any]     ={
    """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/config.json""",
}







class 				snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__      :  str					=							"""gpt_neox_japanese"""




						def __init__(				self   ,						__lowercase=3_2_0_0_0   ,						__lowercase=2_5_6_0   ,						__lowercase=3_2   ,						__lowercase=3_2   ,						__lowercase=4   ,						__lowercase="gelu"   ,						__lowercase=1.00   ,						__lowercase=1_0_0_0_0   ,						__lowercase=2_0_4_8   ,						__lowercase=0.02   ,						__lowercase=1e-5   ,						__lowercase=True   ,						__lowercase=3_1_9_9_6   ,						__lowercase=3_1_9_9_9   ,						__lowercase=0.1   ,						__lowercase=0.0   ,						**__lowercase   ,						)					->    str:
										super().__init__(bos_token_id=__lowercase   ,						eos_token_id=__lowercase   ,						**__lowercase							)
										lowerCAmelCase_ :      Optional[Any]    				=				vocab_size
										lowerCAmelCase_ :      Tuple    				=				max_position_embeddings
										lowerCAmelCase_ :      Optional[Any]    				=				hidden_size
										lowerCAmelCase_ :      Optional[Any]    				=				num_hidden_layers
										lowerCAmelCase_ :      str    				=				num_attention_heads
										lowerCAmelCase_ :      str    				=				intermediate_multiple_size
										lowerCAmelCase_ :      str    				=				hidden_act
										lowerCAmelCase_ :      Dict    				=				rotary_pct
										lowerCAmelCase_ :      Union[str, Any]    				=				rotary_emb_base
										lowerCAmelCase_ :      int    				=				initializer_range
										lowerCAmelCase_ :      Any    				=				layer_norm_eps
										lowerCAmelCase_ :      Optional[Any]    				=				use_cache
										lowerCAmelCase_ :      Tuple    				=				attention_dropout
										lowerCAmelCase_ :      Dict    				=				hidden_dropout