

from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings




@maybe_allow_in_graph
class      UpperCamelCase (     nn.Module   ):

    def __init__(			self :      Optional[int]							,    snake_case__ :      int							,    snake_case__ :      int							,    snake_case__ :      int							,    snake_case__ :      Optional[int]=0.0							,    snake_case__ :      Optional[int] = None							,    snake_case__ :      str = "geglu"							,    snake_case__ :      Optional[int] = None							,    snake_case__ :      bool = False							,    snake_case__ :      bool = False							,    snake_case__ :      bool = False							,    snake_case__ :      bool = False							,    snake_case__ :      bool = True							,    snake_case__ :      str = "layer_norm"							,    snake_case__ :      bool = False							,    ):

     """simple docstring"""






     super().__init__()
     SCREAMING_SNAKE_CASE								=			only_cross_attention

     SCREAMING_SNAKE_CASE								=			(num_embeds_ada_norm is not None) and norm_type == 'ada_norm_zero'
     SCREAMING_SNAKE_CASE								=			(num_embeds_ada_norm is not None) and norm_type == 'ada_norm'

     if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
      raise ValueError(
          F"""`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"""
          F""" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."""    )

     # Define 3 blocks. Each block has its own normalization layer.
     # 1. Self-Attn
     if self.use_ada_layer_norm:
      SCREAMING_SNAKE_CASE								=			AdaLayerNorm(snake_case__							,    snake_case__    )
     elif self.use_ada_layer_norm_zero:
      SCREAMING_SNAKE_CASE								=			AdaLayerNormZero(snake_case__							,    snake_case__    )
     else:
      SCREAMING_SNAKE_CASE								=			nn.LayerNorm(snake_case__							,    elementwise_affine=snake_case__    )
     SCREAMING_SNAKE_CASE								=			Attention(
         query_dim=snake_case__							,    heads=snake_case__							,    dim_head=snake_case__							,    dropout=snake_case__							,    bias=snake_case__							,    cross_attention_dim=cross_attention_dim if only_cross_attention else None							,    upcast_attention=snake_case__							,    )

     # 2. Cross-Attn
     if cross_attention_dim is not None or double_self_attention:
      # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
      # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
      # the second cross attention block.
      SCREAMING_SNAKE_CASE								=			(
          AdaLayerNorm(snake_case__							,    snake_case__    )
          if self.use_ada_layer_norm
          else nn.LayerNorm(snake_case__							,    elementwise_affine=snake_case__    )
      )
      SCREAMING_SNAKE_CASE								=			Attention(
          query_dim=snake_case__							,    cross_attention_dim=cross_attention_dim if not double_self_attention else None							,    heads=snake_case__							,    dim_head=snake_case__							,    dropout=snake_case__							,    bias=snake_case__							,    upcast_attention=snake_case__							,    )  # is self-attn if encoder_hidden_states is none
     else:
      SCREAMING_SNAKE_CASE								=			None
      SCREAMING_SNAKE_CASE								=			None

     # 3. Feed-forward
     SCREAMING_SNAKE_CASE								=			nn.LayerNorm(snake_case__							,    elementwise_affine=snake_case__    )
     SCREAMING_SNAKE_CASE								=			FeedForward(snake_case__							,    dropout=snake_case__							,    activation_fn=snake_case__							,    final_dropout=snake_case__    )

     # let chunk size default to None
     SCREAMING_SNAKE_CASE								=			None
     SCREAMING_SNAKE_CASE								=			0

    def UpperCamelCase			(			self :      Union[str, Any]							,    snake_case__ :      Optional[int]							,    snake_case__ :      int    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			chunk_size
     SCREAMING_SNAKE_CASE								=			dim




    def UpperCamelCase			(			self :      Tuple							,    snake_case__ :      torch.FloatTensor							,    snake_case__ :      Optional[torch.FloatTensor] = None							,    snake_case__ :      Optional[torch.FloatTensor] = None							,    snake_case__ :      Optional[torch.FloatTensor] = None							,    snake_case__ :      Optional[torch.LongTensor] = None							,    snake_case__ :      Dict[str, Any] = None							,    snake_case__ :      Optional[torch.LongTensor] = None							,    ):

     """simple docstring"""






     if self.use_ada_layer_norm:
      SCREAMING_SNAKE_CASE								=			self.norma(snake_case__							,    snake_case__    )
     elif self.use_ada_layer_norm_zero:
      SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE								=			self.norma(
          snake_case__							,    snake_case__							,    snake_case__							,    hidden_dtype=hidden_states.dtype    )
     else:
      SCREAMING_SNAKE_CASE								=			self.norma(snake_case__    )

     SCREAMING_SNAKE_CASE								=			cross_attention_kwargs if cross_attention_kwargs is not None else {}

     SCREAMING_SNAKE_CASE								=			self.attna(
         snake_case__							,    encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None							,    attention_mask=snake_case__							,    **snake_case__							,    )
     if self.use_ada_layer_norm_zero:
      SCREAMING_SNAKE_CASE								=			gate_msa.unsqueeze(1    ) * attn_output
     SCREAMING_SNAKE_CASE								=			attn_output + hidden_states

     # 2. Cross-Attention
     if self.attna is not None:
      SCREAMING_SNAKE_CASE								=			(
          self.norma(snake_case__							,    snake_case__    ) if self.use_ada_layer_norm else self.norma(snake_case__    )
      )

      SCREAMING_SNAKE_CASE								=			self.attna(
          snake_case__							,    encoder_hidden_states=snake_case__							,    attention_mask=snake_case__							,    **snake_case__							,    )
      SCREAMING_SNAKE_CASE								=			attn_output + hidden_states

     # 3. Feed-forward
     SCREAMING_SNAKE_CASE								=			self.norma(snake_case__    )

     if self.use_ada_layer_norm_zero:
      SCREAMING_SNAKE_CASE								=			norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

     if self._chunk_size is not None:
      # "feed_forward_chunk_size" can be used to save memory
      if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
       raise ValueError(
           F"""`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."""    )

      SCREAMING_SNAKE_CASE								=			norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
      SCREAMING_SNAKE_CASE								=			torch.cat(
          [self.ff(snake_case__    ) for hid_slice in norm_hidden_states.chunk(snake_case__							,    dim=self._chunk_dim    )]							,    dim=self._chunk_dim							,    )
     else:
      SCREAMING_SNAKE_CASE								=			self.ff(snake_case__    )

     if self.use_ada_layer_norm_zero:
      SCREAMING_SNAKE_CASE								=			gate_mlp.unsqueeze(1    ) * ff_output

     SCREAMING_SNAKE_CASE								=			ff_output + hidden_states

     return hidden_states




class      UpperCamelCase (     nn.Module   ):

    def __init__(			self :      Union[str, Any]							,    snake_case__ :      int							,    snake_case__ :      Optional[int] = None							,    snake_case__ :      int = 4							,    snake_case__ :      float = 0.0							,    snake_case__ :      str = "geglu"							,    snake_case__ :      bool = False							,    ):

     """simple docstring"""






     super().__init__()
     SCREAMING_SNAKE_CASE								=			int(dim * mult    )
     SCREAMING_SNAKE_CASE								=			dim_out if dim_out is not None else dim

     if activation_fn == "gelu":
      SCREAMING_SNAKE_CASE								=			GELU(snake_case__							,    snake_case__    )
     if activation_fn == "gelu-approximate":
      SCREAMING_SNAKE_CASE								=			GELU(snake_case__							,    snake_case__							,    approximate='tanh'    )
     elif activation_fn == "geglu":
      SCREAMING_SNAKE_CASE								=			GEGLU(snake_case__							,    snake_case__    )
     elif activation_fn == "geglu-approximate":
      SCREAMING_SNAKE_CASE								=			ApproximateGELU(snake_case__							,    snake_case__    )

     SCREAMING_SNAKE_CASE								=			nn.ModuleList([]    )
     # project in
     self.net.append(snake_case__    )
     # project dropout
     self.net.append(nn.Dropout(snake_case__    )    )
     # project out
     self.net.append(nn.Linear(snake_case__							,    snake_case__    )    )
     # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
     if final_dropout:
      self.net.append(nn.Dropout(snake_case__    )    )




    def UpperCamelCase			(			self :      List[str]							,    snake_case__ :      int    ):

     """simple docstring"""






     for module in self.net:
      SCREAMING_SNAKE_CASE								=			module(snake_case__    )
     return hidden_states




class      UpperCamelCase (     nn.Module   ):

    def __init__(			self :      Dict							,    snake_case__ :      int							,    snake_case__ :      int							,    snake_case__ :      str = "none"    ):

     """simple docstring"""






     super().__init__()
     SCREAMING_SNAKE_CASE								=			nn.Linear(snake_case__							,    snake_case__    )
     SCREAMING_SNAKE_CASE								=			approximate

    def UpperCamelCase			(			self :      str							,    snake_case__ :      Any    ):

     """simple docstring"""






     if gate.device.type != "mps":
      return F.gelu(snake_case__							,    approximate=self.approximate    )
     # mps: gelu is not implemented for float16
     return F.gelu(gate.to(dtype=torch.floataa    )							,    approximate=self.approximate    ).to(dtype=gate.dtype    )




    def UpperCamelCase			(			self :      List[str]							,    snake_case__ :      Tuple    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.proj(snake_case__    )
     SCREAMING_SNAKE_CASE								=			self.gelu(snake_case__    )
     return hidden_states




class      UpperCamelCase (     nn.Module   ):

    def __init__(			self :      Optional[int]							,    snake_case__ :      int							,    snake_case__ :      int    ):

     """simple docstring"""






     super().__init__()
     SCREAMING_SNAKE_CASE								=			nn.Linear(snake_case__							,    dim_out * 2    )

    def UpperCamelCase			(			self :      int							,    snake_case__ :      List[str]    ):

     """simple docstring"""






     if gate.device.type != "mps":
      return F.gelu(snake_case__    )
     # mps: gelu is not implemented for float16
     return F.gelu(gate.to(dtype=torch.floataa    )    ).to(dtype=gate.dtype    )




    def UpperCamelCase			(			self :      Union[str, Any]							,    snake_case__ :      Optional[int]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE								=			self.proj(snake_case__    ).chunk(2							,    dim=-1    )
     return hidden_states * self.gelu(snake_case__    )




class      UpperCamelCase (     nn.Module   ):

    def __init__(			self :      List[str]							,    snake_case__ :      int							,    snake_case__ :      int    ):

     """simple docstring"""






     super().__init__()
     SCREAMING_SNAKE_CASE								=			nn.Linear(snake_case__							,    snake_case__    )




    def UpperCamelCase			(			self :      Optional[Any]							,    snake_case__ :      Tuple    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.proj(snake_case__    )
     return x * torch.sigmoid(1.702 * x    )




class      UpperCamelCase (     nn.Module   ):

    def __init__(			self :      Any							,    snake_case__ :      Optional[Any]							,    snake_case__ :      Any    ):

     """simple docstring"""






     super().__init__()
     SCREAMING_SNAKE_CASE								=			nn.Embedding(snake_case__							,    snake_case__    )
     SCREAMING_SNAKE_CASE								=			nn.SiLU()
     SCREAMING_SNAKE_CASE								=			nn.Linear(snake_case__							,    embedding_dim * 2    )
     SCREAMING_SNAKE_CASE								=			nn.LayerNorm(snake_case__							,    elementwise_affine=snake_case__    )




    def UpperCamelCase			(			self :      List[Any]							,    snake_case__ :      Dict							,    snake_case__ :      int    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.linear(self.silu(self.emb(snake_case__    )    )    )
     SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE								=			torch.chunk(snake_case__							,    2    )
     SCREAMING_SNAKE_CASE								=			self.norm(snake_case__    ) * (1 + scale) + shift
     return x




class      UpperCamelCase (     nn.Module   ):

    def __init__(			self :      Optional[int]							,    snake_case__ :      int							,    snake_case__ :      Optional[int]    ):

     """simple docstring"""






     super().__init__()

     SCREAMING_SNAKE_CASE								=			CombinedTimestepLabelEmbeddings(snake_case__							,    snake_case__    )

     SCREAMING_SNAKE_CASE								=			nn.SiLU()
     SCREAMING_SNAKE_CASE								=			nn.Linear(snake_case__							,    6 * embedding_dim							,    bias=snake_case__    )
     SCREAMING_SNAKE_CASE								=			nn.LayerNorm(snake_case__							,    elementwise_affine=snake_case__							,    eps=1E-6    )




    def UpperCamelCase			(			self :      Optional[Any]							,    snake_case__ :      Tuple							,    snake_case__ :      int							,    snake_case__ :      Dict							,    snake_case__ :      Any=None    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.linear(self.silu(self.emb(snake_case__							,    snake_case__							,    hidden_dtype=snake_case__    )    )    )
     SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE								=			emb.chunk(6							,    dim=1    )
     SCREAMING_SNAKE_CASE								=			self.norm(snake_case__    ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
     return x, gate_msa, shift_mlp, scale_mlp, gate_mlp




class      UpperCamelCase (     nn.Module   ):

    def __init__(			self :      Dict							,    snake_case__ :      int							,    snake_case__ :      int							,    snake_case__ :      int							,    snake_case__ :      Optional[str] = None							,    snake_case__ :      float = 1E-5    ):

     """simple docstring"""






     super().__init__()
     SCREAMING_SNAKE_CASE								=			num_groups
     SCREAMING_SNAKE_CASE								=			eps

     if act_fn is None:
      SCREAMING_SNAKE_CASE								=			None
     else:
      SCREAMING_SNAKE_CASE								=			get_activation(snake_case__    )

     SCREAMING_SNAKE_CASE								=			nn.Linear(snake_case__							,    out_dim * 2    )




    def UpperCamelCase			(			self :      Dict							,    snake_case__ :      int							,    snake_case__ :      List[str]    ):

     """simple docstring"""






     if self.act:
      SCREAMING_SNAKE_CASE								=			self.act(snake_case__    )
     SCREAMING_SNAKE_CASE								=			self.linear(snake_case__    )
     SCREAMING_SNAKE_CASE								=			emb[:, :, None, None]
     SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE								=			emb.chunk(2							,    dim=1    )

     SCREAMING_SNAKE_CASE								=			F.group_norm(snake_case__							,    self.num_groups							,    eps=self.eps    )
     SCREAMING_SNAKE_CASE								=			x * (1 + scale) + shift
     return x


def        __lowerCAmelCase    (       _UpperCamelCase		:  int	)		->							int:
 '''simple docstring'''


 SCREAMING_SNAKE_CASE								=			abs(_UpperCamelCase	)
 SCREAMING_SNAKE_CASE								=			0
 while n > 0:
  res += n % 10
  n //= 10
 return res


def        __lowerCAmelCase    (       _UpperCamelCase		:  int	)		->							int:
 '''simple docstring'''


 SCREAMING_SNAKE_CASE								=			abs(_UpperCamelCase	)
 return n if n < 10 else n % 10 + sum_of_digits(n // 10	)


def        __lowerCAmelCase    (       _UpperCamelCase		:  int	)		->							int:
 '''simple docstring'''


 return sum(int(_UpperCamelCase	) for c in str(abs(_UpperCamelCase	)	)	)


def        __lowerCAmelCase    (       )		->							None:
 '''simple docstring'''


 from collections.abc import Callable
 from timeit import timeit

 def benchmark_a_function(_UpperCamelCase		:  Callable		,							_UpperCamelCase		:  int	) -> None:
  SCREAMING_SNAKE_CASE								=			f"""{func.__name__}({value})"""
  SCREAMING_SNAKE_CASE								=			timeit(f"""__main__.{call}"""		,							setup='import __main__'	)
  print(f"""{call:56} = {func(_UpperCamelCase	)} -- {timing:.4f} seconds"""	)

 for value in (26_21_44, 11_25_89_99_06_84_26_24, 1_26_76_50_60_02_28_22_94_01_49_67_03_20_53_76):
  for func in (sum_of_digits, sum_of_digits_recursion, sum_of_digits_compact):
   benchmark_a_function(_UpperCamelCase		,							_UpperCamelCase	)
  print()


if __name__ == "__main__":
      import doctest

      doctest.testmod()
      benchmark()

