






'''simple docstring'''
import tensorflow as tf

from ...tf_utils import shape_list






class      __SCREAMING_SNAKE_CASE      (			tf.keras.layers.Layer				):




 def __init__(      self	:							Union[str, Any]					,      __lowercase	:							Union[str, Any]					,      __lowercase	:							Tuple					,      __lowercase	:							List[str]					,      __lowercase	:							Union[str, Any]					,      __lowercase	:							Dict=1					,      __lowercase	:							List[Any]=False					,      **__lowercase	:							str      )	->						Optional[Any]:
       super().__init__(**__lowercase      )

       SCREAMING_SNAKE_CASE__     :				List[str]									=vocab_size
       SCREAMING_SNAKE_CASE__     :				Dict									=d_embed
       SCREAMING_SNAKE_CASE__     :				Dict									=d_proj

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=cutoffs + [vocab_size]
       SCREAMING_SNAKE_CASE__     :				List[str]									=[0] + self.cutoffs
       SCREAMING_SNAKE_CASE__     :				str									=div_val

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=self.cutoffs[0]
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=len(self.cutoffs      ) - 1
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=self.shortlist_size + self.n_clusters
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=keep_order

       SCREAMING_SNAKE_CASE__     :				Tuple									=[]
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=[]




 def      __magic_name__	(      self	:							Any					,      __lowercase	:							int      )	->						Tuple:
       if self.n_clusters > 0:
             SCREAMING_SNAKE_CASE__     :				Any									=self.add_weight(
                 shape=(self.n_clusters, self.d_embed)					,      initializer='''zeros'''					,      trainable=__lowercase					,      name='''cluster_weight'''      )
             SCREAMING_SNAKE_CASE__     :				Optional[Any]									=self.add_weight(
                 shape=(self.n_clusters,)					,      initializer='''zeros'''					,      trainable=__lowercase					,      name='''cluster_bias'''      )

       if self.div_val == 1:
             for i in range(len(self.cutoffs      )      ):
                   if self.d_proj != self.d_embed:
                         SCREAMING_SNAKE_CASE__     :				str									=self.add_weight(
                             shape=(self.d_embed, self.d_proj)					,      initializer='''zeros'''					,      trainable=__lowercase					,      name=F"out_projs_._{i}"					,      )
                         self.out_projs.append(__lowercase      )
                   else:
                         self.out_projs.append(__lowercase      )
                   SCREAMING_SNAKE_CASE__     :				int									=self.add_weight(
                       shape=(self.vocab_size, self.d_embed)					,      initializer='''zeros'''					,      trainable=__lowercase					,      name=F"out_layers_._{i}_._weight"					,      )
                   SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=self.add_weight(
                       shape=(self.vocab_size,)					,      initializer='''zeros'''					,      trainable=__lowercase					,      name=F"out_layers_._{i}_._bias"					,      )
                   self.out_layers.append((weight, bias)      )
       else:
             for i in range(len(self.cutoffs      )      ):
                   SCREAMING_SNAKE_CASE__		,	SCREAMING_SNAKE_CASE__     :				Optional[Any]									=self.cutoff_ends[i], self.cutoff_ends[i + 1]
                   SCREAMING_SNAKE_CASE__     :				str									=self.d_embed // (self.div_val**i)

                   SCREAMING_SNAKE_CASE__     :				Any									=self.add_weight(
                       shape=(d_emb_i, self.d_proj)					,      initializer='''zeros'''					,      trainable=__lowercase					,      name=F"out_projs_._{i}"      )
                   self.out_projs.append(__lowercase      )
                   SCREAMING_SNAKE_CASE__     :				int									=self.add_weight(
                       shape=(r_idx - l_idx, d_emb_i)					,      initializer='''zeros'''					,      trainable=__lowercase					,      name=F"out_layers_._{i}_._weight"					,      )
                   SCREAMING_SNAKE_CASE__     :				Dict									=self.add_weight(
                       shape=(r_idx - l_idx,)					,      initializer='''zeros'''					,      trainable=__lowercase					,      name=F"out_layers_._{i}_._bias"					,      )
                   self.out_layers.append((weight, bias)      )
       super().build(__lowercase      )




 @staticmethod
 def      __magic_name__	(      __lowercase	:							List[str]					,      __lowercase	:							Tuple					,      __lowercase	:							Any					,      __lowercase	:							str=None      )	->						Optional[Any]:
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=x
       if proj is not None:
             SCREAMING_SNAKE_CASE__     :				List[Any]									=tf.einsum('''ibd,ed->ibe'''					,      __lowercase					,      __lowercase      )
       return tf.einsum('''ibd,nd->ibn'''					,      __lowercase					,      __lowercase      ) + b




 @staticmethod
 def      __magic_name__	(      __lowercase	:							List[Any]					,      __lowercase	:							List[Any]      )	->						List[Any]:
       SCREAMING_SNAKE_CASE__     :				List[str]									=shape_list(__lowercase      )
       SCREAMING_SNAKE_CASE__     :				List[str]									=tf.range(lp_size[0]					,      dtype=target.dtype      )
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=tf.stack([r, target]					,      1      )
       return tf.gather_nd(__lowercase					,      __lowercase      )




 def      __magic_name__	(      self	:							Optional[int]					,      __lowercase	:							int					,      __lowercase	:							Union[str, Any]					,      __lowercase	:							str=True					,      __lowercase	:							int=False      )	->						Dict:
       SCREAMING_SNAKE_CASE__     :				List[str]									=0
       if self.n_clusters == 0:
             SCREAMING_SNAKE_CASE__     :				Optional[Any]									=self._logit(__lowercase					,      self.out_layers[0][0]					,      self.out_layers[0][1]					,      self.out_projs[0]      )
             if target is not None:
                   SCREAMING_SNAKE_CASE__     :				Optional[int]									=tf.nn.sparse_softmax_cross_entropy_with_logits(labels=__lowercase					,      logits=__lowercase      )
             SCREAMING_SNAKE_CASE__     :				Tuple									=tf.nn.log_softmax(__lowercase					,      axis=-1      )
       else:
             SCREAMING_SNAKE_CASE__     :				Optional[Any]									=shape_list(__lowercase      )
             SCREAMING_SNAKE_CASE__     :				Dict									=[]
             SCREAMING_SNAKE_CASE__     :				Any									=tf.zeros(hidden_sizes[:2]      )
             for i in range(len(self.cutoffs      )      ):
                   SCREAMING_SNAKE_CASE__		,	SCREAMING_SNAKE_CASE__     :				List[Any]									=self.cutoff_ends[i], self.cutoff_ends[i + 1]
                   if target is not None:
                         SCREAMING_SNAKE_CASE__     :				List[Any]									=(target >= l_idx) & (target < r_idx)
                         SCREAMING_SNAKE_CASE__     :				List[Any]									=tf.where(__lowercase      )
                         SCREAMING_SNAKE_CASE__     :				Optional[Any]									=tf.boolean_mask(__lowercase					,      __lowercase      ) - l_idx

                   if self.div_val == 1:
                         SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=self.out_layers[0][0][l_idx:r_idx]
                         SCREAMING_SNAKE_CASE__     :				Dict									=self.out_layers[0][1][l_idx:r_idx]
                   else:
                         SCREAMING_SNAKE_CASE__     :				Dict									=self.out_layers[i][0]
                         SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=self.out_layers[i][1]

                   if i == 0:
                         SCREAMING_SNAKE_CASE__     :				Any									=tf.concat([cur_W, self.cluster_weight]					,      0      )
                         SCREAMING_SNAKE_CASE__     :				int									=tf.concat([cur_b, self.cluster_bias]					,      0      )

                         SCREAMING_SNAKE_CASE__     :				int									=self._logit(__lowercase					,      __lowercase					,      __lowercase					,      self.out_projs[0]      )
                         SCREAMING_SNAKE_CASE__     :				Optional[int]									=tf.nn.log_softmax(__lowercase      )
                         out.append(head_logprob[..., : self.cutoffs[0]]      )
                         if target is not None:
                               SCREAMING_SNAKE_CASE__     :				int									=tf.boolean_mask(__lowercase					,      __lowercase      )
                               SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=self._gather_logprob(__lowercase					,      __lowercase      )
                   else:
                         SCREAMING_SNAKE_CASE__     :				Optional[int]									=self._logit(__lowercase					,      __lowercase					,      __lowercase					,      self.out_projs[i]      )
                         SCREAMING_SNAKE_CASE__     :				int									=tf.nn.log_softmax(__lowercase      )
                         SCREAMING_SNAKE_CASE__     :				Dict									=self.cutoffs[0] + i - 1  # No probability for the head cluster
                         SCREAMING_SNAKE_CASE__     :				Optional[int]									=head_logprob[..., cluster_prob_idx, None] + tail_logprob
                         out.append(__lowercase      )
                         if target is not None:
                               SCREAMING_SNAKE_CASE__     :				int									=tf.boolean_mask(__lowercase					,      __lowercase      )
                               SCREAMING_SNAKE_CASE__     :				int									=tf.boolean_mask(__lowercase					,      __lowercase      )
                               SCREAMING_SNAKE_CASE__     :				Any									=self._gather_logprob(__lowercase					,      __lowercase      )
                               cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
                   if target is not None:
                         loss += tf.scatter_nd(__lowercase					,      -cur_logprob					,      shape_list(__lowercase      )      )
             SCREAMING_SNAKE_CASE__     :				Optional[Any]									=tf.concat(__lowercase					,      axis=-1      )

       if target is not None:
             if return_mean:
                   SCREAMING_SNAKE_CASE__     :				Dict									=tf.reduce_mean(__lowercase      )
             # Add the training-time loss value to the layer using `self.add_loss()`.
             self.add_loss(__lowercase      )

             # Log the loss as a metric (we could log arbitrary metrics,
             # including different metrics for training and inference.
             self.add_metric(__lowercase					,      name=self.name					,      aggregation='''mean''' if return_mean else ''''''      )

       return out







'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_					= logging.get_logger(__name__)

a_					= {
    'EleutherAI/gpt-neox-20b': 'https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json',
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}






class      __SCREAMING_SNAKE_CASE      (			lowerCamelCase				):
 snake_case_  							=							"""gpt_neox"""




 def __init__(      self	:							List[Any]					,      __lowercase	:							Union[str, Any]=5_04_32					,      __lowercase	:							int=61_44					,      __lowercase	:							Tuple=44					,      __lowercase	:							List[str]=64					,      __lowercase	:							str=2_45_76					,      __lowercase	:							Dict="gelu"					,      __lowercase	:							Tuple=0.25					,      __lowercase	:							Tuple=1_00_00					,      __lowercase	:							Tuple=0.0					,      __lowercase	:							str=0.0					,      __lowercase	:							List[Any]=0.1					,      __lowercase	:							Dict=20_48					,      __lowercase	:							Any=0.02					,      __lowercase	:							Dict=1e-5					,      __lowercase	:							List[Any]=True					,      __lowercase	:							str=0					,      __lowercase	:							Optional[Any]=2					,      __lowercase	:							Tuple=False					,      __lowercase	:							List[Any]=True					,      __lowercase	:							Optional[Any]=None					,      **__lowercase	:							Any					,      )	->						Dict:
       super().__init__(bos_token_id=__lowercase					,      eos_token_id=__lowercase					,      **__lowercase      )
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=vocab_size
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=max_position_embeddings
       SCREAMING_SNAKE_CASE__     :				Any									=hidden_size
       SCREAMING_SNAKE_CASE__     :				str									=num_hidden_layers
       SCREAMING_SNAKE_CASE__     :				Any									=num_attention_heads
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=intermediate_size
       SCREAMING_SNAKE_CASE__     :				Dict									=hidden_act
       SCREAMING_SNAKE_CASE__     :				str									=rotary_pct
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=rotary_emb_base
       SCREAMING_SNAKE_CASE__     :				List[Any]									=attention_dropout
       SCREAMING_SNAKE_CASE__     :				List[Any]									=hidden_dropout
       SCREAMING_SNAKE_CASE__     :				str									=classifier_dropout
       SCREAMING_SNAKE_CASE__     :				Any									=initializer_range
       SCREAMING_SNAKE_CASE__     :				Dict									=layer_norm_eps
       SCREAMING_SNAKE_CASE__     :				Any									=use_cache
       SCREAMING_SNAKE_CASE__     :				Tuple									=tie_word_embeddings
       SCREAMING_SNAKE_CASE__     :				Tuple									=use_parallel_residual
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=rope_scaling
       self._rope_scaling_validation()

       if self.hidden_size % self.num_attention_heads != 0:
             raise ValueError(
                 '''The hidden size is not divisble by the number of attention heads! Make sure to update them!'''      )




 def      __magic_name__	(      self	:							Optional[Any]      )	->						Optional[Any]:
       if self.rope_scaling is None:
             return

       if not isinstance(self.rope_scaling					,      __lowercase      ) or len(self.rope_scaling      ) != 2:
             raise ValueError(
                 '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
                 F"got {self.rope_scaling}"      )
       SCREAMING_SNAKE_CASE__     :				int									=self.rope_scaling.get('''type'''					,      __lowercase      )
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=self.rope_scaling.get('''factor'''					,      __lowercase      )
       if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
             raise ValueError(
                 F"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"      )
       if rope_scaling_factor is None or not isinstance(__lowercase					,      __lowercase      ) or rope_scaling_factor <= 1.0:
             raise ValueError(F"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"      )