


from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase				:  Optional[Any]      					=							logging.get_logger(__name__)

__UpperCamelCase				:  Any      					=							{
    'EleutherAI/gpt-neox-20b': 'https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json',
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     str			= 'gpt_neox'





    def __init__(	self				:   Tuple       ,   __snake_case				:   Optional[int]=50_432       ,   __snake_case				:   int=6_144       ,   __snake_case				:   str=44       ,   __snake_case				:   int=64       ,   __snake_case				:   str=24_576       ,   __snake_case				:   Union[str, Any]="gelu"       ,   __snake_case				:   Tuple=0.25       ,   __snake_case				:   Tuple=10_000       ,   __snake_case				:   str=0.0       ,   __snake_case				:   Optional[Any]=0.0       ,   __snake_case				:   List[Any]=0.1       ,   __snake_case				:   str=2_048       ,   __snake_case				:   List[Any]=0.02       ,   __snake_case				:   Optional[int]=1E-5       ,   __snake_case				:   str=True       ,   __snake_case				:   int=0       ,   __snake_case				:   Tuple=2       ,   __snake_case				:   Tuple=False       ,   __snake_case				:   Optional[int]=True       ,   __snake_case				:   Tuple=None       ,   **__snake_case				:   Any       ,   ):





       '''simple docstring'''
       super().__init__(bos_token_id=__snake_case       ,   eos_token_id=__snake_case       ,   **__snake_case	)
       UpperCAmelCase_							:      Dict						=   vocab_size
       UpperCAmelCase_							:      Optional[int]						=   max_position_embeddings
       UpperCAmelCase_							:      Tuple						=   hidden_size
       UpperCAmelCase_							:      List[str]						=   num_hidden_layers
       UpperCAmelCase_							:      Any						=   num_attention_heads
       UpperCAmelCase_							:      Optional[Any]						=   intermediate_size
       UpperCAmelCase_							:      List[str]						=   hidden_act
       UpperCAmelCase_							:      Dict						=   rotary_pct
       UpperCAmelCase_							:      Optional[int]						=   rotary_emb_base
       UpperCAmelCase_							:      str						=   attention_dropout
       UpperCAmelCase_							:      List[Any]						=   hidden_dropout
       UpperCAmelCase_							:      str						=   classifier_dropout
       UpperCAmelCase_							:      int						=   initializer_range
       UpperCAmelCase_							:      List[str]						=   layer_norm_eps
       UpperCAmelCase_							:      Any						=   use_cache
       UpperCAmelCase_							:      str						=   tie_word_embeddings
       UpperCAmelCase_							:      Dict						=   use_parallel_residual
       UpperCAmelCase_							:      Any						=   rope_scaling
       self._rope_scaling_validation()

       if self.hidden_size % self.num_attention_heads != 0:
          raise ValueError(
              '''The hidden size is not divisble by the number of attention heads! Make sure to update them!'''	)





    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       if self.rope_scaling is None:
          return

       if not isinstance(self.rope_scaling       ,   __snake_case	) or len(self.rope_scaling	) != 2:
          raise ValueError(
              '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
              f'''got {self.rope_scaling}'''	)
       UpperCAmelCase_							:      int						=   self.rope_scaling.get('''type'''       ,   __snake_case	)
       UpperCAmelCase_							:      Any						=   self.rope_scaling.get('''factor'''       ,   __snake_case	)
       if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
          raise ValueError(
              f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''	)
       if rope_scaling_factor is None or not isinstance(__snake_case       ,   __snake_case	) or rope_scaling_factor <= 1.0:
          raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''	)



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__UpperCamelCase				:  str      					=							{
    'configuration_x_clip': [
        'XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'XCLIPConfig',
        'XCLIPTextConfig',
        'XCLIPVisionConfig',
    ],
    'processing_x_clip': ['XCLIPProcessor'],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __UpperCamelCase				:  Any      					=							[
           'XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST',
           'XCLIPModel',
           'XCLIPPreTrainedModel',
           'XCLIPTextModel',
           'XCLIPVisionModel',
       ]

if TYPE_CHECKING:
       from .configuration_x_clip import (
           XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
           XCLIPConfig,
           XCLIPTextConfig,
           XCLIPVisionConfig,
       )
       from .processing_x_clip import XCLIPProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_x_clip import (
                  XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
                  XCLIPModel,
                  XCLIPPreTrainedModel,
                  XCLIPTextModel,
                  XCLIPVisionModel,
              )

else:
       import sys

       __UpperCamelCase				:  List[str]      					=							_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)