

'''simple docstring'''







import re
from typing import Callable, List, Optional, Union

import tensorflow as tf


try:
      from tensorflow.keras.optimizers.legacy import Adam
except ImportError:
      from tensorflow.keras.optimizers import Adam


class 					A__      (      tf.keras.optimizers.schedules.LearningRateSchedule			):



    def __init__(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__ = 1.0		,				UpperCamelCase__ = None		,				)						->      List[str]:





           '''simple docstring'''
           super().__init__()
           A_						=      initial_learning_rate
           A_						=      warmup_steps
           A_						=      power
           A_						=      decay_schedule_fn
           A_						=      name



    def __call__(					self		,				UpperCamelCase__       )						->      Union[str, Any]:





           '''simple docstring'''
           with tf.name_scope(self.name or """WarmUp"""       ) as name:
                  # Implements polynomial warmup. i.e., if global_step < warmup_steps, the
                  # learning rate will be `global_step/num_warmup_steps * init_lr`.
                  A_						=      tf.cast(UpperCamelCase__		,				tf.floataa       )
                  A_						=      tf.cast(self.warmup_steps		,				tf.floataa       )
                  A_						=      global_step_float / warmup_steps_float
                  A_						=      self.initial_learning_rate * tf.math.pow(UpperCamelCase__		,				self.power       )
                  return tf.cond(
                      global_step_float < warmup_steps_float		,				lambda: warmup_learning_rate		,				lambda: self.decay_schedule_fn(step - self.warmup_steps       )		,				name=UpperCamelCase__		,				)







    def 					snake_case_			(					self       )						->      List[str]:





           '''simple docstring'''
           return {
               "initial_learning_rate": self.initial_learning_rate,
               "decay_schedule_fn": self.decay_schedule_fn,
               "warmup_steps": self.warmup_steps,
               "power": self.power,
               "name": self.name,
           }

def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__,       UpperCAmelCase__,       UpperCAmelCase__ = 0.0,       UpperCAmelCase__ = 0.9,       UpperCAmelCase__ = 0.999,       UpperCAmelCase__ = 1e-8,       UpperCAmelCase__ = None,       UpperCAmelCase__ = None,       UpperCAmelCase__ = 0.0,       UpperCAmelCase__ = 1.0,       UpperCAmelCase__ = None,       )			->			int:
       A_						=      tf.keras.optimizers.schedules.PolynomialDecay(
           initial_learning_rate=UpperCAmelCase__,       decay_steps=num_train_steps - num_warmup_steps,       end_learning_rate=init_lr * min_lr_ratio,       power=UpperCAmelCase__,       )
       if num_warmup_steps:
              A_						=      WarmUp(
                  initial_learning_rate=UpperCAmelCase__,       decay_schedule_fn=UpperCAmelCase__,       warmup_steps=UpperCAmelCase__,       )
       if weight_decay_rate > 0.0:
              A_						=      AdamWeightDecay(
                  learning_rate=UpperCAmelCase__,       weight_decay_rate=UpperCAmelCase__,       beta_a=UpperCAmelCase__,       beta_a=UpperCAmelCase__,       epsilon=UpperCAmelCase__,       clipnorm=UpperCAmelCase__,       global_clipnorm=UpperCAmelCase__,       exclude_from_weight_decay=["""LayerNorm""", """layer_norm""", """bias"""],       include_in_weight_decay=UpperCAmelCase__,       )
       else:
              A_						=      tf.keras.optimizers.Adam(
                  learning_rate=UpperCAmelCase__,       beta_a=UpperCAmelCase__,       beta_a=UpperCAmelCase__,       epsilon=UpperCAmelCase__,       clipnorm=UpperCAmelCase__,       global_clipnorm=UpperCAmelCase__,       )
       # We return the optimizer and the LR scheduler in order to better track the
       # evolution of the LR independently of the optimizer.
       return optimizer, lr_schedule


class 					A__      (      _snake_case			):



    def __init__(					self		,				UpperCamelCase__ = 0.001		,				UpperCamelCase__ = 0.9		,				UpperCamelCase__ = 0.999		,				UpperCamelCase__ = 1e-7		,				UpperCamelCase__ = False		,				UpperCamelCase__ = 0.0		,				UpperCamelCase__ = None		,				UpperCamelCase__ = None		,				UpperCamelCase__ = "AdamWeightDecay"		,				**UpperCamelCase__		,				)						->      Optional[Any]:





           '''simple docstring'''
           super().__init__(UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				**UpperCamelCase__       )
           A_						=      weight_decay_rate
           A_						=      include_in_weight_decay
           A_						=      exclude_from_weight_decay



    @classmethod
    def 					snake_case_			(					cls		,				UpperCamelCase__       )						->      Union[str, Any]:





           '''simple docstring'''
           A_						=      {"""WarmUp""": WarmUp}
           return super(UpperCamelCase__		,				cls       ).from_config(UpperCamelCase__		,				custom_objects=UpperCamelCase__       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )						->      Any:





           '''simple docstring'''
           super(UpperCamelCase__		,				self       )._prepare_local(UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )
           A_						=      tf.constant(
               self.weight_decay_rate		,				name="""adam_weight_decay_rate"""       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )						->      List[Any]:





           '''simple docstring'''
           A_						=      self._do_use_weight_decay(var.name       )
           if do_decay:
                  return var.assign_sub(
                      learning_rate * var * apply_state[(var.device, var.dtype.base_dtype)]["""weight_decay_rate"""]		,				use_locking=self._use_locking		,				)
           return tf.no_op()



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__=None		,				**UpperCamelCase__       )						->      int:





           '''simple docstring'''
           A_					,			A_						=      list(zip(*UpperCamelCase__       )       )
           return super(UpperCamelCase__		,				self       ).apply_gradients(zip(UpperCamelCase__		,				UpperCamelCase__       )		,				name=UpperCamelCase__		,				**UpperCamelCase__       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )						->      Union[str, Any]:





           '''simple docstring'''
           if apply_state is None:
                  return self._decayed_lr_t[var_dtype], {}

           A_						=      apply_state or {}
           A_						=      apply_state.get((var_device, var_dtype)       )
           if coefficients is None:
                  A_						=      self._fallback_apply_state(UpperCamelCase__		,				UpperCamelCase__       )
                  A_						=      coefficients

           return coefficients["lr_t"], {"apply_state": apply_state}



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__=None       )						->      List[Any]:





           '''simple docstring'''
           A_					,			A_						=      self._get_lr(var.device		,				var.dtype.base_dtype		,				UpperCamelCase__       )
           A_						=      self._decay_weights_op(UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )
           with tf.control_dependencies([decay]       ):
                  return super(UpperCamelCase__		,				self       )._resource_apply_dense(UpperCamelCase__		,				UpperCamelCase__		,				**UpperCamelCase__       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__=None       )						->      Any:





           '''simple docstring'''
           A_					,			A_						=      self._get_lr(var.device		,				var.dtype.base_dtype		,				UpperCamelCase__       )
           A_						=      self._decay_weights_op(UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )
           with tf.control_dependencies([decay]       ):
                  return super(UpperCamelCase__		,				self       )._resource_apply_sparse(UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				**UpperCamelCase__       )



    def 					snake_case_			(					self       )						->      Optional[int]:





           '''simple docstring'''
           A_						=      super().get_config()
           config.update({"""weight_decay_rate""": self.weight_decay_rate}       )
           return config







    def 					snake_case_			(					self		,				UpperCamelCase__       )						->      str:





           '''simple docstring'''
           if self.weight_decay_rate == 0:
                  return False

           if self._include_in_weight_decay:
                  for r in self._include_in_weight_decay:
                         if re.search(UpperCamelCase__		,				UpperCamelCase__       ) is not None:
                                return True

           if self._exclude_from_weight_decay:
                  for r in self._exclude_from_weight_decay:
                         if re.search(UpperCamelCase__		,				UpperCamelCase__       ) is not None:
                                return False
           return True


class 					A__      (      _snake_case			):



    def __init__(					self       )						->      List[Any]:





           '''simple docstring'''
           A_						=      []
           A_						=      None



    @property
    def 					snake_case_			(					self       )						->      List[str]:





           '''simple docstring'''
           if self._accum_steps is None:
                  A_						=      tf.Variable(
                      tf.constant(0		,				dtype=tf.intaa       )		,				trainable=UpperCamelCase__		,				synchronization=tf.VariableSynchronization.ON_READ		,				aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA		,				)

           return self._accum_steps.value()



    @property
    def 					snake_case_			(					self       )						->      int:





           '''simple docstring'''
           if not self._gradients:
                  raise ValueError("""The accumulator should be called first to initialize the gradients"""       )
           return [gradient.value() if gradient is not None else gradient for gradient in self._gradients]



    def __call__(					self		,				UpperCamelCase__       )						->      int:





           '''simple docstring'''
           if not self._gradients:
                  A_						=      self.step  # Create the step variable.
                  self._gradients.extend(
                      [
                          tf.Variable(
                              tf.zeros_like(UpperCamelCase__       )		,				trainable=UpperCamelCase__		,				synchronization=tf.VariableSynchronization.ON_READ		,				aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA		,				)
                          if gradient is not None
                          else gradient
                          for gradient in gradients
                      ]       )
           if len(UpperCamelCase__       ) != len(self._gradients       ):
                  raise ValueError(f'''Expected {len(self._gradients       )} gradients, but got {len(UpperCamelCase__       )}'''       )

           for accum_gradient, gradient in zip(self._gradients		,				UpperCamelCase__       ):
                  if accum_gradient is not None and gradient is not None:
                         accum_gradient.assign_add(UpperCamelCase__       )

           self._accum_steps.assign_add(1       )







    def 					snake_case_			(					self       )						->      Dict:





           '''simple docstring'''
           if not self._gradients:
                  return
           self._accum_steps.assign(0       )
           for gradient in self._gradients:
                  if gradient is not None:
                         gradient.assign(tf.zeros_like(UpperCamelCase__       )       )



'''simple docstring'''







import tensorflow as tf

from ...tf_utils import shape_list


class 					A__      (      tf.keras.layers.Layer			):



    def __init__(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__=1		,				UpperCamelCase__=False		,				**UpperCamelCase__       )						->      Union[str, Any]:





           '''simple docstring'''
           super().__init__(**UpperCamelCase__       )

           A_						=      vocab_size
           A_						=      d_embed
           A_						=      d_proj

           A_						=      cutoffs + [vocab_size]
           A_						=      [0] + self.cutoffs
           A_						=      div_val

           A_						=      self.cutoffs[0]
           A_						=      len(self.cutoffs       ) - 1
           A_						=      self.shortlist_size + self.n_clusters
           A_						=      keep_order

           A_						=      []
           A_						=      []



    def 					snake_case_			(					self		,				UpperCamelCase__       )						->      Union[str, Any]:





           '''simple docstring'''
           if self.n_clusters > 0:
                  A_						=      self.add_weight(
                      shape=(self.n_clusters, self.d_embed)		,				initializer="""zeros"""		,				trainable=UpperCamelCase__		,				name="""cluster_weight"""       )
                  A_						=      self.add_weight(
                      shape=(self.n_clusters,)		,				initializer="""zeros"""		,				trainable=UpperCamelCase__		,				name="""cluster_bias"""       )

           if self.div_val == 1:
                  for i in range(len(self.cutoffs       )       ):
                         if self.d_proj != self.d_embed:
                                A_						=      self.add_weight(
                                    shape=(self.d_embed, self.d_proj)		,				initializer="""zeros"""		,				trainable=UpperCamelCase__		,				name=f'''out_projs_._{i}'''		,				)
                                self.out_projs.append(UpperCamelCase__       )
                         else:
                                self.out_projs.append(UpperCamelCase__       )
                         A_						=      self.add_weight(
                             shape=(self.vocab_size, self.d_embed)		,				initializer="""zeros"""		,				trainable=UpperCamelCase__		,				name=f'''out_layers_._{i}_._weight'''		,				)
                         A_						=      self.add_weight(
                             shape=(self.vocab_size,)		,				initializer="""zeros"""		,				trainable=UpperCamelCase__		,				name=f'''out_layers_._{i}_._bias'''		,				)
                         self.out_layers.append((weight, bias)       )
           else:
                  for i in range(len(self.cutoffs       )       ):
                         A_					,			A_						=      self.cutoff_ends[i], self.cutoff_ends[i + 1]
                         A_						=      self.d_embed // (self.div_val**i)

                         A_						=      self.add_weight(
                             shape=(d_emb_i, self.d_proj)		,				initializer="""zeros"""		,				trainable=UpperCamelCase__		,				name=f'''out_projs_._{i}'''       )
                         self.out_projs.append(UpperCamelCase__       )
                         A_						=      self.add_weight(
                             shape=(r_idx - l_idx, d_emb_i)		,				initializer="""zeros"""		,				trainable=UpperCamelCase__		,				name=f'''out_layers_._{i}_._weight'''		,				)
                         A_						=      self.add_weight(
                             shape=(r_idx - l_idx,)		,				initializer="""zeros"""		,				trainable=UpperCamelCase__		,				name=f'''out_layers_._{i}_._bias'''		,				)
                         self.out_layers.append((weight, bias)       )
           super().build(UpperCamelCase__       )



    @staticmethod
    def 					snake_case_			(					UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__=None       )						->      List[Any]:





           '''simple docstring'''
           A_						=      x
           if proj is not None:
                  A_						=      tf.einsum("""ibd,ed->ibe"""		,				UpperCamelCase__		,				UpperCamelCase__       )
           return tf.einsum("""ibd,nd->ibn"""		,				UpperCamelCase__		,				UpperCamelCase__       ) + b



    @staticmethod
    def 					snake_case_			(					UpperCamelCase__		,				UpperCamelCase__       )						->      List[str]:





           '''simple docstring'''
           A_						=      shape_list(UpperCamelCase__       )
           A_						=      tf.range(lp_size[0]		,				dtype=target.dtype       )
           A_						=      tf.stack([r, target]		,				1       )
           return tf.gather_nd(UpperCamelCase__		,				UpperCamelCase__       )







    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__=True		,				UpperCamelCase__=False       )						->      Optional[int]:





           '''simple docstring'''
           A_						=      0
           if self.n_clusters == 0:
                  A_						=      self._logit(UpperCamelCase__		,				self.out_layers[0][0]		,				self.out_layers[0][1]		,				self.out_projs[0]       )
                  if target is not None:
                         A_						=      tf.nn.sparse_softmax_cross_entropy_with_logits(labels=UpperCamelCase__		,				logits=UpperCamelCase__       )
                  A_						=      tf.nn.log_softmax(UpperCamelCase__		,				axis=-1       )
           else:
                  A_						=      shape_list(UpperCamelCase__       )
                  A_						=      []
                  A_						=      tf.zeros(hidden_sizes[:2]       )
                  for i in range(len(self.cutoffs       )       ):
                         A_					,			A_						=      self.cutoff_ends[i], self.cutoff_ends[i + 1]
                         if target is not None:
                                A_						=      (target >= l_idx) & (target < r_idx)
                                A_						=      tf.where(UpperCamelCase__       )
                                A_						=      tf.boolean_mask(UpperCamelCase__		,				UpperCamelCase__       ) - l_idx

                         if self.div_val == 1:
                                A_						=      self.out_layers[0][0][l_idx:r_idx]
                                A_						=      self.out_layers[0][1][l_idx:r_idx]
                         else:
                                A_						=      self.out_layers[i][0]
                                A_						=      self.out_layers[i][1]

                         if i == 0:
                                A_						=      tf.concat([cur_W, self.cluster_weight]		,				0       )
                                A_						=      tf.concat([cur_b, self.cluster_bias]		,				0       )

                                A_						=      self._logit(UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				self.out_projs[0]       )
                                A_						=      tf.nn.log_softmax(UpperCamelCase__       )
                                out.append(head_logprob[..., : self.cutoffs[0]]       )
                                if target is not None:
                                       A_						=      tf.boolean_mask(UpperCamelCase__		,				UpperCamelCase__       )
                                       A_						=      self._gather_logprob(UpperCamelCase__		,				UpperCamelCase__       )
                         else:
                                A_						=      self._logit(UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				self.out_projs[i]       )
                                A_						=      tf.nn.log_softmax(UpperCamelCase__       )
                                A_						=      self.cutoffs[0] + i - 1  # No probability for the head cluster
                                A_						=      head_logprob[..., cluster_prob_idx, None] + tail_logprob
                                out.append(UpperCamelCase__       )
                                if target is not None:
                                       A_						=      tf.boolean_mask(UpperCamelCase__		,				UpperCamelCase__       )
                                       A_						=      tf.boolean_mask(UpperCamelCase__		,				UpperCamelCase__       )
                                       A_						=      self._gather_logprob(UpperCamelCase__		,				UpperCamelCase__       )
                                       cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
                         if target is not None:
                                loss += tf.scatter_nd(UpperCamelCase__		,				-cur_logprob		,				shape_list(UpperCamelCase__       )       )
                  A_						=      tf.concat(UpperCamelCase__		,				axis=-1       )

           if target is not None:
                  if return_mean:
                         A_						=      tf.reduce_mean(UpperCamelCase__       )
                  # Add the training-time loss value to the layer using `self.add_loss()`.
                  self.add_loss(UpperCamelCase__       )

                  # Log the loss as a metric (we could log arbitrary metrics,
                  # including different metrics for training and inference.
                  self.add_metric(UpperCamelCase__		,				name=self.name		,				aggregation="""mean""" if return_mean else """"""       )

           return out
