





import torch
from torch import nn







class   __lowerCAmelCase			(					nn.Module  ):





   def __init__( self					,			snake_case					,			snake_case					,			snake_case					,			snake_case					,			snake_case=1					,			snake_case=False      )				->  List[Any]:
      """simple docstring"""





      super().__init__()

      a__	:						List[str]      		=  n_token
      a__	:						Optional[Any]      		=  d_embed
      a__	:						str      		=  d_proj

      a__	:						Optional[int]      		=  cutoffs + [n_token]
      a__	:						Union[str, Any]      		=  [0] + self.cutoffs
      a__	:						Optional[int]      		=  div_val

      a__	:						List[str]      		=  self.cutoffs[0]
      a__	:						Dict      		=  len(self.cutoffs      ) - 1
      a__	:						Optional[Any]      		=  self.shortlist_size + self.n_clusters

      if self.n_clusters > 0:
         a__	:						str      		=  nn.Parameter(torch.zeros(self.n_clusters					,			self.d_embed      )      )
         a__	:						List[str]      		=  nn.Parameter(torch.zeros(self.n_clusters      )      )

      a__	:						Union[str, Any]      		=  nn.ModuleList()
      a__	:						List[Any]      		=  nn.ParameterList()

      if div_val == 1:
         for i in range(len(self.cutoffs      )      ):
            if d_proj != d_embed:
               self.out_projs.append(nn.Parameter(torch.FloatTensor(snake_case					,			snake_case      )      )      )
            else:
               self.out_projs.append(snake_case      )

         self.out_layers.append(nn.Linear(snake_case					,			snake_case      )      )
      else:
         for i in range(len(self.cutoffs      )      ):
            a__		,     a__	:						List[str]      		=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
            a__	:						Tuple      		=  d_embed // (div_val**i)

            self.out_projs.append(nn.Parameter(torch.FloatTensor(snake_case					,			snake_case      )      )      )

            self.out_layers.append(nn.Linear(snake_case					,			r_idx - l_idx      )      )

      a__	:						Optional[Any]      		=  keep_order





   def 	_snake_case ( self					,			snake_case					,			snake_case					,			snake_case					,			snake_case      )				->  Any:
      """simple docstring"""





      if proj is None:
         a__	:						Optional[Any]      		=  nn.functional.linear(snake_case					,			snake_case					,			bias=snake_case      )
      else:
         # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
         a__	:						Optional[int]      		=  nn.functional.linear(snake_case					,			proj.t().contiguous()      )
         a__	:						Tuple      		=  nn.functional.linear(snake_case					,			snake_case					,			bias=snake_case      )
         # else:
         #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
         #     if bias is not None:
         #         logit = logit + bias

      return logit





   def 	_snake_case ( self					,			snake_case					,			snake_case=None					,			snake_case=False      )				->  Optional[Any]:
      """simple docstring"""





      if labels is not None:
         # Shift so that tokens < n predict n
         a__	:						Optional[Any]      		=  hidden[..., :-1, :].contiguous()
         a__	:						Optional[int]      		=  labels[..., 1:].contiguous()
         a__	:						Optional[Any]      		=  hidden.view(-1					,			hidden.size(-1      )      )
         a__	:						List[Any]      		=  labels.view(-1      )
         if hidden.size(0      ) != labels.size(0      ):
            raise RuntimeError("Input and labels should have the same size in the batch dimension."      )
      else:
         a__	:						Any      		=  hidden.view(-1					,			hidden.size(-1      )      )

      if self.n_clusters == 0:
         a__	:						Dict      		=  self._compute_logit(snake_case					,			self.out_layers[0].weight					,			self.out_layers[0].bias					,			self.out_projs[0]      )
         if labels is not None:
            a__	:						List[Any]      		=  labels != -100
            a__	:						str      		=  torch.zeros_like(snake_case					,			dtype=hidden.dtype					,			device=hidden.device      )
            a__	:						Any      		=  (
                -nn.functional.log_softmax(snake_case					,			dim=-1      )[mask].gather(1					,			labels[mask].unsqueeze(1      )      ).squeeze(1      )
            )
         else:
            a__	:						int      		=  nn.functional.log_softmax(snake_case					,			dim=-1      )
      else:
         # construct weights and biases
         a__		,     a__	:						str      		=  [], []
         for i in range(len(self.cutoffs      )      ):
            if self.div_val == 1:
               a__		,     a__	:						Dict      		=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
               a__	:						Any      		=  self.out_layers[0].weight[l_idx:r_idx]
               a__	:						Optional[Any]      		=  self.out_layers[0].bias[l_idx:r_idx]
            else:
               a__	:						str      		=  self.out_layers[i].weight
               a__	:						List[str]      		=  self.out_layers[i].bias

            if i == 0:
               a__	:						int      		=  torch.cat([weight_i, self.cluster_weight]					,			dim=0      )
               a__	:						List[Any]      		=  torch.cat([bias_i, self.cluster_bias]					,			dim=0      )

            weights.append(snake_case      )
            biases.append(snake_case      )

         a__		,     a__		,     a__	:						Tuple      		=  weights[0], biases[0], self.out_projs[0]

         a__	:						Any      		=  self._compute_logit(snake_case					,			snake_case					,			snake_case					,			snake_case      )
         a__	:						str      		=  nn.functional.log_softmax(snake_case					,			dim=1      )

         if labels is None:
            a__	:						Dict      		=  hidden.new_empty((head_logit.size(0      ), self.n_token)      )
         else:
            a__	:						Optional[int]      		=  torch.zeros_like(snake_case					,			dtype=hidden.dtype					,			device=hidden.device      )

         a__	:						Tuple      		=  0
         a__	:						List[str]      		=  [0] + self.cutoffs
         for i in range(len(snake_case      ) - 1      ):
            a__		,     a__	:						Optional[int]      		=  cutoff_values[i], cutoff_values[i + 1]

            if labels is not None:
               a__	:						Any      		=  (labels >= l_idx) & (labels < r_idx)
               a__	:						int      		=  mask_i.nonzero().squeeze()

               if indices_i.numel() == 0:
                  continue

               a__	:						int      		=  labels.index_select(0					,			snake_case      ) - l_idx
               a__	:						Optional[Any]      		=  head_logprob.index_select(0					,			snake_case      )
               a__	:						Union[str, Any]      		=  hidden.index_select(0					,			snake_case      )
            else:
               a__	:						Any      		=  hidden

            if i == 0:
               if labels is not None:
                  a__	:						Optional[int]      		=  head_logprob_i.gather(1					,			target_i[:, None]      ).squeeze(1      )
               else:
                  a__	:						Optional[Any]      		=  head_logprob[:, : self.cutoffs[0]]
            else:
               a__		,     a__		,     a__	:						str      		=  weights[i], biases[i], self.out_projs[i]

               a__	:						str      		=  self._compute_logit(snake_case					,			snake_case					,			snake_case					,			snake_case      )
               a__	:						List[Any]      		=  nn.functional.log_softmax(snake_case					,			dim=1      )
               a__	:						List[str]      		=  self.cutoffs[0] + i - 1  # No probability for the head cluster
               if labels is not None:
                  a__	:						Union[str, Any]      		=  head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                      1					,			target_i[:, None]      ).squeeze(1      )
               else:
                  a__	:						List[Any]      		=  head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                  a__	:						List[Any]      		=  logprob_i

            if labels is not None:
               if (hasattr(self					,			"keep_order"      ) and self.keep_order) or keep_order:
                  out.index_copy_(0					,			snake_case					,			-logprob_i      )
               else:
                  out[offset : offset + logprob_i.size(0      )].copy_(-logprob_i      )
               offset += logprob_i.size(0      )

      return out





   def 	_snake_case ( self					,			snake_case      )				->  List[Any]:
      """simple docstring"""





      if self.n_clusters == 0:
         a__	:						Any      		=  self._compute_logit(snake_case					,			self.out_layers[0].weight					,			self.out_layers[0].bias					,			self.out_projs[0]      )
         return nn.functional.log_softmax(snake_case					,			dim=-1      )
      else:
         # construct weights and biases
         a__		,     a__	:						str      		=  [], []
         for i in range(len(self.cutoffs      )      ):
            if self.div_val == 1:
               a__		,     a__	:						List[Any]      		=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
               a__	:						Any      		=  self.out_layers[0].weight[l_idx:r_idx]
               a__	:						str      		=  self.out_layers[0].bias[l_idx:r_idx]
            else:
               a__	:						int      		=  self.out_layers[i].weight
               a__	:						Optional[Any]      		=  self.out_layers[i].bias

            if i == 0:
               a__	:						Tuple      		=  torch.cat([weight_i, self.cluster_weight]					,			dim=0      )
               a__	:						Union[str, Any]      		=  torch.cat([bias_i, self.cluster_bias]					,			dim=0      )

            weights.append(snake_case      )
            biases.append(snake_case      )

         a__		,     a__		,     a__	:						List[Any]      		=  weights[0], biases[0], self.out_projs[0]
         a__	:						List[Any]      		=  self._compute_logit(snake_case					,			snake_case					,			snake_case					,			snake_case      )

         a__	:						List[str]      		=  hidden.new_empty((head_logit.size(0      ), self.n_token)      )
         a__	:						Optional[Any]      		=  nn.functional.log_softmax(snake_case					,			dim=1      )

         a__	:						Dict      		=  [0] + self.cutoffs
         for i in range(len(snake_case      ) - 1      ):
            a__		,     a__	:						Optional[Any]      		=  cutoff_values[i], cutoff_values[i + 1]

            if i == 0:
               a__	:						Any      		=  head_logprob[:, : self.cutoffs[0]]
            else:
               a__		,     a__		,     a__	:						Optional[Any]      		=  weights[i], biases[i], self.out_projs[i]

               a__	:						Tuple      		=  self._compute_logit(snake_case					,			snake_case					,			snake_case					,			snake_case      )
               a__	:						List[Any]      		=  nn.functional.log_softmax(snake_case					,			dim=1      )

               a__	:						Union[str, Any]      		=  head_logprob[:, -i] + tail_logprob_i
               a__	:						Dict      		=  logprob_i

         return out









# Lint as: python3
import itertools
import os
import re


SCREAMING_SNAKE_CASE__       :			Tuple			      =						re.compile(R"""([A-Z]+)([A-Z][a-z])""")
SCREAMING_SNAKE_CASE__       :			List[str]			      =						re.compile(R"""([a-z\d])([A-Z])""")

SCREAMING_SNAKE_CASE__       :			Tuple			      =						re.compile(R"""(?<!_)_(?!_)""")
SCREAMING_SNAKE_CASE__       :			Dict			      =						re.compile(R"""(_{2,})""")

SCREAMING_SNAKE_CASE__       :			List[Any]			      =						R"""^\w+(\.\w+)*$"""

SCREAMING_SNAKE_CASE__       :			Union[str, Any]			      =						R"""<>:/\|?*"""






def       _A  (      lowerCamelCase   ):
   a__	:						List[str]      		=  _uppercase_uppercase_re.sub(r"\1_\2"							, lowerCamelCase   )
   a__	:						Dict      		=  _lowercase_uppercase_re.sub(r"\1_\2"							, lowerCamelCase   )
   return name.lower()






def       _A  (      lowerCamelCase   ):
   a__	:						Tuple      		=  _single_underscore_re.split(lowerCamelCase   )
   a__	:						Any      		=  [_multiple_underscores_re.split(lowerCamelCase   ) for n in name]
   return "".join(n.capitalize() for n in itertools.chain.from_iterable(lowerCamelCase   ) if n != ""   )






def       _A  (      lowerCamelCase   ):
   if os.path.basename(lowerCamelCase   ) != name:
      raise ValueError(F"""Should be a dataset name, not a path: {name}"""   )
   return camelcase_to_snakecase(lowerCamelCase   )






def       _A  (      lowerCamelCase							, lowerCamelCase   ):
   if os.path.basename(lowerCamelCase   ) != name:
      raise ValueError(F"""Should be a dataset name, not a path: {name}"""   )
   if not re.match(_split_re							, lowerCamelCase   ):
      raise ValueError(F"""Split name should match '{_split_re}'' but got '{split}'."""   )
   return F"""{filename_prefix_for_name(lowerCamelCase   )}-{split}"""






def       _A  (      lowerCamelCase							, lowerCamelCase							, lowerCamelCase							, lowerCamelCase=None   ):
   a__	:						Union[str, Any]      		=  filename_prefix_for_split(lowerCamelCase							, lowerCamelCase   )
   if filetype_suffix:
      prefix += F""".{filetype_suffix}"""
   a__	:						Any      		=  os.path.join(lowerCamelCase							, lowerCamelCase   )
   return F"""{filepath}*"""






def       _A  (      lowerCamelCase							, lowerCamelCase							, lowerCamelCase							, lowerCamelCase=None							, lowerCamelCase=None   ):
   a__	:						List[str]      		=  filename_prefix_for_split(lowerCamelCase							, lowerCamelCase   )
   a__	:						Tuple      		=  os.path.join(lowerCamelCase							, lowerCamelCase   )

   if shard_lengths:
      a__	:						List[str]      		=  len(lowerCamelCase   )
      a__	:						str      		=  [F"""{prefix}-{shard_id:05d}-of-{num_shards:05d}""" for shard_id in range(lowerCamelCase   )]
      if filetype_suffix:
         a__	:						Optional[Any]      		=  [filename + F""".{filetype_suffix}""" for filename in filenames]
      return filenames
   else:
      a__	:						Optional[int]      		=  prefix
      if filetype_suffix:
         filename += F""".{filetype_suffix}"""
      return [filename]

