import torch
from torch import nn



class  SCREAMING_SNAKE_CASE_ (					nn.Module			):
  '''simple docstring'''


  def __init__(				self    :					str      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]      ,	SCREAMING_SNAKE_CASE__    :					int      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					Dict      ,	SCREAMING_SNAKE_CASE__    :					str=1      ,	SCREAMING_SNAKE_CASE__    :					List[Any]=False     )    ->				List[Any]:
         super().__init__()

         A						:     Tuple         =n_token
         A						:     Any         =d_embed
         A						:     Dict         =d_proj

         A						:     List[Any]         =cutoffs + [n_token]
         A						:     str         =[0] + self.cutoffs
         A						:     int         =div_val

         A						:     Optional[int]         =self.cutoffs[0]
         A						:     Any         =len(self.cutoffs     ) - 1
         A						:     Optional[int]         =self.shortlist_size + self.n_clusters

         if self.n_clusters > 0:
                A						:     Any         =nn.Parameter(torch.zeros(self.n_clusters      ,	self.d_embed     )     )
                A						:     Dict         =nn.Parameter(torch.zeros(self.n_clusters     )     )

         A						:     Any         =nn.ModuleList()
         A						:     str         =nn.ParameterList()

         if div_val == 1:
                for i in range(len(self.cutoffs     )     ):
                       if d_proj != d_embed:
                              self.out_projs.append(nn.Parameter(torch.FloatTensor(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )     )     )
                       else:
                              self.out_projs.append(SCREAMING_SNAKE_CASE__     )

                self.out_layers.append(nn.Linear(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )     )
         else:
                for i in range(len(self.cutoffs     )     ):
                       A  ,  A						:     str         =self.cutoff_ends[i], self.cutoff_ends[i + 1]
                       A						:     List[str]         =d_embed // (div_val**i)

                       self.out_projs.append(nn.Parameter(torch.FloatTensor(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )     )     )

                       self.out_layers.append(nn.Linear(SCREAMING_SNAKE_CASE__      ,	r_idx - l_idx     )     )

         A						:     List[Any]         =keep_order







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					Dict      ,	SCREAMING_SNAKE_CASE__    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					Any     )    ->				Union[str, Any]:
         if proj is None:
                A						:     int         =nn.functional.linear(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	bias=SCREAMING_SNAKE_CASE__     )
         else:
                # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
                A						:     Tuple         =nn.functional.linear(SCREAMING_SNAKE_CASE__      ,	proj.t().contiguous()     )
                A						:     Any         =nn.functional.linear(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	bias=SCREAMING_SNAKE_CASE__     )
                # else:
                #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
                #     if bias is not None:
                #         logit = logit + bias

         return logit







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					Any      ,	SCREAMING_SNAKE_CASE__    :					Tuple=None      ,	SCREAMING_SNAKE_CASE__    :					Dict=False     )    ->				Union[str, Any]:

         if labels is not None:
                # Shift so that tokens < n predict n
                A						:     Tuple         =hidden[..., :-1, :].contiguous()
                A						:     Dict         =labels[..., 1:].contiguous()
                A						:     Optional[int]         =hidden.view(-1      ,	hidden.size(-1     )     )
                A						:     List[Any]         =labels.view(-1     )
                if hidden.size(0     ) != labels.size(0     ):
                       raise RuntimeError('Input and labels should have the same size in the batch dimension.'     )
         else:
                A						:     Dict         =hidden.view(-1      ,	hidden.size(-1     )     )

         if self.n_clusters == 0:
                A						:     Optional[Any]         =self._compute_logit(SCREAMING_SNAKE_CASE__      ,	self.out_layers[0].weight      ,	self.out_layers[0].bias      ,	self.out_projs[0]     )
                if labels is not None:
                       A						:     Any         =labels != -1_00
                       A						:     Union[str, Any]         =torch.zeros_like(SCREAMING_SNAKE_CASE__      ,	dtype=hidden.dtype      ,	device=hidden.device     )
                       A						:     Tuple         =(
                           -nn.functional.log_softmax(SCREAMING_SNAKE_CASE__      ,	dim=-1     )[mask].gather(1      ,	labels[mask].unsqueeze(1     )     ).squeeze(1     )
                       )
                else:
                       A						:     Any         =nn.functional.log_softmax(SCREAMING_SNAKE_CASE__      ,	dim=-1     )
         else:
                # construct weights and biases
                A  ,  A						:     Optional[Any]         =[], []
                for i in range(len(self.cutoffs     )     ):
                       if self.div_val == 1:
                              A  ,  A						:     Optional[int]         =self.cutoff_ends[i], self.cutoff_ends[i + 1]
                              A						:     List[Any]         =self.out_layers[0].weight[l_idx:r_idx]
                              A						:     Optional[Any]         =self.out_layers[0].bias[l_idx:r_idx]
                       else:
                              A						:     str         =self.out_layers[i].weight
                              A						:     List[Any]         =self.out_layers[i].bias

                       if i == 0:
                              A						:     List[Any]         =torch.cat([weight_i, self.cluster_weight]      ,	dim=0     )
                              A						:     List[str]         =torch.cat([bias_i, self.cluster_bias]      ,	dim=0     )

                       weights.append(SCREAMING_SNAKE_CASE__     )
                       biases.append(SCREAMING_SNAKE_CASE__     )

                A  ,  A  ,  A						:     Any         =weights[0], biases[0], self.out_projs[0]

                A						:     int         =self._compute_logit(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
                A						:     List[Any]         =nn.functional.log_softmax(SCREAMING_SNAKE_CASE__      ,	dim=1     )

                if labels is None:
                       A						:     List[str]         =hidden.new_empty((head_logit.size(0     ), self.n_token)     )
                else:
                       A						:     Tuple         =torch.zeros_like(SCREAMING_SNAKE_CASE__      ,	dtype=hidden.dtype      ,	device=hidden.device     )

                A						:     Optional[int]         =0
                A						:     List[str]         =[0] + self.cutoffs
                for i in range(len(SCREAMING_SNAKE_CASE__     ) - 1     ):
                       A  ,  A						:     int         =cutoff_values[i], cutoff_values[i + 1]

                       if labels is not None:
                              A						:     List[Any]         =(labels >= l_idx) & (labels < r_idx)
                              A						:     Any         =mask_i.nonzero().squeeze()

                              if indices_i.numel() == 0:
                                     continue

                              A						:     List[str]         =labels.index_select(0      ,	SCREAMING_SNAKE_CASE__     ) - l_idx
                              A						:     Optional[Any]         =head_logprob.index_select(0      ,	SCREAMING_SNAKE_CASE__     )
                              A						:     Dict         =hidden.index_select(0      ,	SCREAMING_SNAKE_CASE__     )
                       else:
                              A						:     Optional[int]         =hidden

                       if i == 0:
                              if labels is not None:
                                     A						:     Dict         =head_logprob_i.gather(1      ,	target_i[:, None]     ).squeeze(1     )
                              else:
                                     A						:     Dict         =head_logprob[:, : self.cutoffs[0]]
                       else:
                              A  ,  A  ,  A						:     int         =weights[i], biases[i], self.out_projs[i]

                              A						:     Tuple         =self._compute_logit(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
                              A						:     int         =nn.functional.log_softmax(SCREAMING_SNAKE_CASE__      ,	dim=1     )
                              A						:     Any         =self.cutoffs[0] + i - 1  # No probability for the head cluster
                              if labels is not None:
                                     A						:     Tuple         =head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                                         1      ,	target_i[:, None]     ).squeeze(1     )
                              else:
                                     A						:     Dict         =head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                                     A						:     Tuple         =logprob_i

                       if labels is not None:
                              if (hasattr(self      ,	'keep_order'     ) and self.keep_order) or keep_order:
                                     out.index_copy_(0      ,	SCREAMING_SNAKE_CASE__      ,	-logprob_i     )
                              else:
                                     out[offset : offset + logprob_i.size(0     )].copy_(-logprob_i     )
                              offset += logprob_i.size(0     )

         return out







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]      ,	SCREAMING_SNAKE_CASE__    :					List[Any]     )    ->				List[Any]:
         if self.n_clusters == 0:
                A						:     str         =self._compute_logit(SCREAMING_SNAKE_CASE__      ,	self.out_layers[0].weight      ,	self.out_layers[0].bias      ,	self.out_projs[0]     )
                return nn.functional.log_softmax(SCREAMING_SNAKE_CASE__      ,	dim=-1     )
         else:
                # construct weights and biases
                A  ,  A						:     str         =[], []
                for i in range(len(self.cutoffs     )     ):
                       if self.div_val == 1:
                              A  ,  A						:     Tuple         =self.cutoff_ends[i], self.cutoff_ends[i + 1]
                              A						:     Tuple         =self.out_layers[0].weight[l_idx:r_idx]
                              A						:     Optional[Any]         =self.out_layers[0].bias[l_idx:r_idx]
                       else:
                              A						:     Optional[Any]         =self.out_layers[i].weight
                              A						:     Any         =self.out_layers[i].bias

                       if i == 0:
                              A						:     Tuple         =torch.cat([weight_i, self.cluster_weight]      ,	dim=0     )
                              A						:     List[Any]         =torch.cat([bias_i, self.cluster_bias]      ,	dim=0     )

                       weights.append(SCREAMING_SNAKE_CASE__     )
                       biases.append(SCREAMING_SNAKE_CASE__     )

                A  ,  A  ,  A						:     Optional[Any]         =weights[0], biases[0], self.out_projs[0]
                A						:     Optional[int]         =self._compute_logit(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

                A						:     Union[str, Any]         =hidden.new_empty((head_logit.size(0     ), self.n_token)     )
                A						:     Union[str, Any]         =nn.functional.log_softmax(SCREAMING_SNAKE_CASE__      ,	dim=1     )

                A						:     str         =[0] + self.cutoffs
                for i in range(len(SCREAMING_SNAKE_CASE__     ) - 1     ):
                       A  ,  A						:     Any         =cutoff_values[i], cutoff_values[i + 1]

                       if i == 0:
                              A						:     int         =head_logprob[:, : self.cutoffs[0]]
                       else:
                              A  ,  A  ,  A						:     Optional[int]         =weights[i], biases[i], self.out_projs[i]

                              A						:     List[str]         =self._compute_logit(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
                              A						:     List[str]         =nn.functional.log_softmax(SCREAMING_SNAKE_CASE__      ,	dim=1     )

                              A						:     Optional[int]         =head_logprob[:, -i] + tail_logprob_i
                              A						:     Optional[Any]         =logprob_i

                return out







import inspect
import warnings
from typing import Any, Dict, Optional, Union

from packaging import version



def A__  (			*lowercase:     Tuple,		lowercase:     Optional[Union[Dict, Any]] = None,		lowercase:     Dict=True,		lowercase:     Any=2					)				->		List[Any]:
       from .. import __version__

       A						:     Optional[Any]         =take_from
       A						:     Union[str, Any]         =()
       if not isinstance(args[0],		lowercase					):
              A						:     List[str]         =(args,)

       for attribute, version_name, message in args:
              if version.parse(version.parse(lowercase					).base_version					) >= version.parse(lowercase					):
                     raise ValueError(
                         F'The deprecation tuple {(attribute, version_name, message)} should be removed since diffusers\''
                         F' version {__version__} is >= {version_name}'					)

              A						:     Tuple         =None
              if isinstance(lowercase,		lowercase					) and attribute in deprecated_kwargs:
                     values += (deprecated_kwargs.pop(lowercase					),)
                     A						:     Union[str, Any]         =F'The `{attribute}` argument is deprecated and will be removed in version {version_name}.'
              elif hasattr(lowercase,		lowercase					):
                     values += (getattr(lowercase,		lowercase					),)
                     A						:     Optional[Any]         =F'The `{attribute}` attribute is deprecated and will be removed in version {version_name}.'
              elif deprecated_kwargs is None:
                     A						:     List[Any]         =F'`{attribute}` is deprecated and will be removed in version {version_name}.'

              if warning is not None:
                     A						:     List[Any]         =warning + ' ' if standard_warn else ''
                     warnings.warn(warning + message,		lowercase,		stacklevel=lowercase					)

       if isinstance(lowercase,		lowercase					) and len(lowercase					) > 0:
              A						:     Any         =inspect.getouterframes(inspect.currentframe()					)[1]
              A						:     int         =call_frame.filename
              A						:     int         =call_frame.lineno
              A						:     Optional[int]         =call_frame.function
              A  ,  A						:     int         =next(iter(deprecated_kwargs.items()					)					)
              raise TypeError(F'{function} in {filename} line {line_number-1} got an unexpected keyword argument `{key}`'					)

       if len(lowercase					) == 0:
              return
       elif len(lowercase					) == 1:
              return values[0]
       return values






