


'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


_lowerCAmelCase  :		str		     =	{
    'configuration_roberta_prelayernorm': [
        'ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'RobertaPreLayerNormConfig',
        'RobertaPreLayerNormOnnxConfig',
    ],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							_lowerCAmelCase  :		Tuple		     =	[
							    'ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'RobertaPreLayerNormForCausalLM',
							    'RobertaPreLayerNormForMaskedLM',
							    'RobertaPreLayerNormForMultipleChoice',
							    'RobertaPreLayerNormForQuestionAnswering',
							    'RobertaPreLayerNormForSequenceClassification',
							    'RobertaPreLayerNormForTokenClassification',
							    'RobertaPreLayerNormModel',
							    'RobertaPreLayerNormPreTrainedModel',
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							_lowerCAmelCase  :		str		     =	[
							    'TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'TFRobertaPreLayerNormForCausalLM',
							    'TFRobertaPreLayerNormForMaskedLM',
							    'TFRobertaPreLayerNormForMultipleChoice',
							    'TFRobertaPreLayerNormForQuestionAnswering',
							    'TFRobertaPreLayerNormForSequenceClassification',
							    'TFRobertaPreLayerNormForTokenClassification',
							    'TFRobertaPreLayerNormMainLayer',
							    'TFRobertaPreLayerNormModel',
							    'TFRobertaPreLayerNormPreTrainedModel',
							]

try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							_lowerCAmelCase  :		int		     =	[
							    'FlaxRobertaPreLayerNormForCausalLM',
							    'FlaxRobertaPreLayerNormForMaskedLM',
							    'FlaxRobertaPreLayerNormForMultipleChoice',
							    'FlaxRobertaPreLayerNormForQuestionAnswering',
							    'FlaxRobertaPreLayerNormForSequenceClassification',
							    'FlaxRobertaPreLayerNormForTokenClassification',
							    'FlaxRobertaPreLayerNormModel',
							    'FlaxRobertaPreLayerNormPreTrainedModel',
							]


if TYPE_CHECKING:
							from .configuration_roberta_prelayernorm import (
							    ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP,
							    RobertaPreLayerNormConfig,
							    RobertaPreLayerNormOnnxConfig,
							)

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_roberta_prelayernorm import (
														    ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    RobertaPreLayerNormForCausalLM,
														    RobertaPreLayerNormForMaskedLM,
														    RobertaPreLayerNormForMultipleChoice,
														    RobertaPreLayerNormForQuestionAnswering,
														    RobertaPreLayerNormForSequenceClassification,
														    RobertaPreLayerNormForTokenClassification,
														    RobertaPreLayerNormModel,
														    RobertaPreLayerNormPreTrainedModel,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_roberta_prelayernorm import (
														    TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFRobertaPreLayerNormForCausalLM,
														    TFRobertaPreLayerNormForMaskedLM,
														    TFRobertaPreLayerNormForMultipleChoice,
														    TFRobertaPreLayerNormForQuestionAnswering,
														    TFRobertaPreLayerNormForSequenceClassification,
														    TFRobertaPreLayerNormForTokenClassification,
														    TFRobertaPreLayerNormMainLayer,
														    TFRobertaPreLayerNormModel,
														    TFRobertaPreLayerNormPreTrainedModel,
														)

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_roberta_prelayernorm import (
														    FlaxRobertaPreLayerNormForCausalLM,
														    FlaxRobertaPreLayerNormForMaskedLM,
														    FlaxRobertaPreLayerNormForMultipleChoice,
														    FlaxRobertaPreLayerNormForQuestionAnswering,
														    FlaxRobertaPreLayerNormForSequenceClassification,
														    FlaxRobertaPreLayerNormForTokenClassification,
														    FlaxRobertaPreLayerNormModel,
														    FlaxRobertaPreLayerNormPreTrainedModel,
														)

else:
							import sys

							_lowerCAmelCase  :		Optional[int]		     =	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCAmelCase  :		str		     =	logging.get_logger(__name__)

_lowerCAmelCase  :		Tuple		     =	{
    's-JoL/Open-Llama-V1': 'https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json',
}



class  lowerCAmelCase				(				a							):
	_lowerCamelCase   :					Union[str, Any]			      = """open-llama"""



	def __init__(					self ,					snake_case__=10_0000 ,					snake_case__=4096 ,					snake_case__=1_1008 ,					snake_case__=32 ,					snake_case__=32 ,					snake_case__="silu" ,					snake_case__=2048 ,					snake_case__=0.0_2 ,					snake_case__=1e-6 ,					snake_case__=True ,					snake_case__=0 ,					snake_case__=1 ,					snake_case__=2 ,					snake_case__=False ,					snake_case__=True ,					snake_case__=0.1 ,					snake_case__=0.1 ,					snake_case__=True ,					snake_case__=True ,					snake_case__=None ,					**snake_case__ ,					):
		lowerCAmelCase					:    Tuple      							= vocab_size
		lowerCAmelCase					:    Optional[Any]      							= max_position_embeddings
		lowerCAmelCase					:    List[Any]      							= hidden_size
		lowerCAmelCase					:    List[Any]      							= intermediate_size
		lowerCAmelCase					:    Tuple      							= num_hidden_layers
		lowerCAmelCase					:    List[Any]      							= num_attention_heads
		lowerCAmelCase					:    List[Any]      							= hidden_act
		lowerCAmelCase					:    Union[str, Any]      							= initializer_range
		lowerCAmelCase					:    str      							= rms_norm_eps
		lowerCAmelCase					:    Optional[int]      							= use_cache
		lowerCAmelCase					:    Dict      							= kwargs.pop(
		    'use_memorry_efficient_attention' ,					snake_case__    )
		lowerCAmelCase					:    Optional[int]      							= hidden_dropout_prob
		lowerCAmelCase					:    Optional[Any]      							= attention_dropout_prob
		lowerCAmelCase					:    Union[str, Any]      							= use_stable_embedding
		lowerCAmelCase					:    Tuple      							= shared_input_output_embedding
		lowerCAmelCase					:    Tuple      							= rope_scaling
		self._rope_scaling_validation()

		super().__init__(
		    pad_token_id=snake_case__ ,					bos_token_id=snake_case__ ,					eos_token_id=snake_case__ ,					tie_word_embeddings=snake_case__ ,					**snake_case__ ,					)



	def 				lowercase     (					self    ):
		if self.rope_scaling is None:
			return

		if not isinstance(self.rope_scaling ,					snake_case__    ) or len(self.rope_scaling    ) != 2:
			raise ValueError(
			    '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
			    f"got {self.rope_scaling}"    )
		lowerCAmelCase					:    List[Any]      							= self.rope_scaling.get('type' ,					snake_case__    )
		lowerCAmelCase					:    List[str]      							= self.rope_scaling.get('factor' ,					snake_case__    )
		if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
			raise ValueError(
			    f"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"    )
		if rope_scaling_factor is None or not isinstance(snake_case__ ,					snake_case__    ) or rope_scaling_factor <= 1.0:
			raise ValueError(f"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"    )
