"""simple docstring"""




from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings
@maybe_allow_in_graph
class 		__A							(				nn.Module	):






     def __init__(	self       ,  a__       ,  a__       ,  a__       ,  a__=0.0       ,  a__ = None       ,  a__ = "geglu"       ,  a__ = None       ,  a__ = False       ,  a__ = False       ,  a__ = False       ,  a__ = False       ,  a__ = True       ,  a__ = "layer_norm"       ,  a__ = False       ,  ):
       super().__init__()
       _lowerCAmelCase		: Optional[Any]						    =  only_cross_attention

       _lowerCAmelCase		: Tuple						    =  (num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
       _lowerCAmelCase		: Optional[int]						    =  (num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

       if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
         raise ValueError(
             F"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
             F" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."						)

       # Define 3 blocks. Each block has its own normalization layer.
       # 1. Self-Attn
       if self.use_ada_layer_norm:
         _lowerCAmelCase		: Optional[Any]						    =  AdaLayerNorm(a__       ,  a__						)
       elif self.use_ada_layer_norm_zero:
         _lowerCAmelCase		: List[Any]						    =  AdaLayerNormZero(a__       ,  a__						)
       else:
         _lowerCAmelCase		: Any						    =  nn.LayerNorm(a__       ,  elementwise_affine=a__						)
       _lowerCAmelCase		: Tuple						    =  Attention(
           query_dim=a__       ,  heads=a__       ,  dim_head=a__       ,  dropout=a__       ,  bias=a__       ,  cross_attention_dim=cross_attention_dim if only_cross_attention else None       ,  upcast_attention=a__       ,  )

       # 2. Cross-Attn
       if cross_attention_dim is not None or double_self_attention:
         # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
         # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
         # the second cross attention block.
         _lowerCAmelCase		: Union[str, Any]						    =  (
             AdaLayerNorm(a__       ,  a__						)
             if self.use_ada_layer_norm
             else nn.LayerNorm(a__       ,  elementwise_affine=a__						)
         )
         _lowerCAmelCase		: Dict						    =  Attention(
             query_dim=a__       ,  cross_attention_dim=cross_attention_dim if not double_self_attention else None       ,  heads=a__       ,  dim_head=a__       ,  dropout=a__       ,  bias=a__       ,  upcast_attention=a__       ,  )  # is self-attn if encoder_hidden_states is none
       else:
         _lowerCAmelCase		: Optional[int]						    =  None
         _lowerCAmelCase		: Optional[int]						    =  None

       # 3. Feed-forward
       _lowerCAmelCase		: Optional[int]						    =  nn.LayerNorm(a__       ,  elementwise_affine=a__						)
       _lowerCAmelCase		: int						    =  FeedForward(a__       ,  dropout=a__       ,  activation_fn=a__       ,  final_dropout=a__						)

       # let chunk size default to None
       _lowerCAmelCase		: str						    =  None
       _lowerCAmelCase		: Optional[Any]						    =  0






     def 					__A						(	self       ,  a__       ,  a__						):
       # Sets chunk feed-forward
       _lowerCAmelCase		: int						    =  chunk_size
       _lowerCAmelCase		: List[str]						    =  dim






     def 					__A						(	self       ,  a__       ,  a__ = None       ,  a__ = None       ,  a__ = None       ,  a__ = None       ,  a__ = None       ,  a__ = None       ,  ):
       # Notice that normalization is always applied before the real computation in the following blocks.
       # 1. Self-Attention
       if self.use_ada_layer_norm:
         _lowerCAmelCase		: Tuple						    =  self.norma(a__       ,  a__						)
       elif self.use_ada_layer_norm_zero:
         _lowerCAmelCase    ,			_lowerCAmelCase    ,			_lowerCAmelCase    ,			_lowerCAmelCase    ,			_lowerCAmelCase		: List[Any]						    =  self.norma(
             a__       ,  a__       ,  a__       ,  hidden_dtype=hidden_states.dtype						)
       else:
         _lowerCAmelCase		: str						    =  self.norma(a__						)

       _lowerCAmelCase		: Optional[Any]						    =  cross_attention_kwargs if cross_attention_kwargs is not None else {}

       _lowerCAmelCase		: List[Any]						    =  self.attna(
           a__       ,  encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None       ,  attention_mask=a__       ,  **a__       ,  )
       if self.use_ada_layer_norm_zero:
         _lowerCAmelCase		: str						    =  gate_msa.unsqueeze(1						) * attn_output
       _lowerCAmelCase		: Dict						    =  attn_output + hidden_states

       # 2. Cross-Attention
       if self.attna is not None:
         _lowerCAmelCase		: Any						    =  (
             self.norma(a__       ,  a__						) if self.use_ada_layer_norm else self.norma(a__						)
         )

         _lowerCAmelCase		: Any						    =  self.attna(
             a__       ,  encoder_hidden_states=a__       ,  attention_mask=a__       ,  **a__       ,  )
         _lowerCAmelCase		: List[Any]						    =  attn_output + hidden_states

       # 3. Feed-forward
       _lowerCAmelCase		: str						    =  self.norma(a__						)

       if self.use_ada_layer_norm_zero:
         _lowerCAmelCase		: int						    =  norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

       if self._chunk_size is not None:
         # "feed_forward_chunk_size" can be used to save memory
         if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
           raise ValueError(
               F"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."						)

         _lowerCAmelCase		: Optional[Any]						    =  norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
         _lowerCAmelCase		: str						    =  torch.cat(
             [self.ff(a__						) for hid_slice in norm_hidden_states.chunk(a__       ,  dim=self._chunk_dim						)]       ,  dim=self._chunk_dim       ,  )
       else:
         _lowerCAmelCase		: Tuple						    =  self.ff(a__						)

       if self.use_ada_layer_norm_zero:
         _lowerCAmelCase		: Optional[int]						    =  gate_mlp.unsqueeze(1						) * ff_output

       _lowerCAmelCase		: List[Any]						    =  ff_output + hidden_states

       return hidden_states







class 		__A							(				nn.Module	):






     def __init__(	self       ,  a__       ,  a__ = None       ,  a__ = 4       ,  a__ = 0.0       ,  a__ = "geglu"       ,  a__ = False       ,  ):
       super().__init__()
       _lowerCAmelCase		: Union[str, Any]						    =  int(dim * mult						)
       _lowerCAmelCase		: Optional[Any]						    =  dim_out if dim_out is not None else dim

       if activation_fn == "gelu":
         _lowerCAmelCase		: Optional[int]						    =  GELU(a__       ,  a__						)
       if activation_fn == "gelu-approximate":
         _lowerCAmelCase		: Any						    =  GELU(a__       ,  a__       ,  approximate="""tanh"""						)
       elif activation_fn == "geglu":
         _lowerCAmelCase		: int						    =  GEGLU(a__       ,  a__						)
       elif activation_fn == "geglu-approximate":
         _lowerCAmelCase		: Any						    =  ApproximateGELU(a__       ,  a__						)

       _lowerCAmelCase		: List[str]						    =  nn.ModuleList([]						)
       # project in
       self.net.append(a__						)
       # project dropout
       self.net.append(nn.Dropout(a__						)						)
       # project out
       self.net.append(nn.Linear(a__       ,  a__						)						)
       # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
       if final_dropout:
         self.net.append(nn.Dropout(a__						)						)






     def 					__A						(	self       ,  a__						):
       for module in self.net:
         _lowerCAmelCase		: Dict						    =  module(a__						)
       return hidden_states







class 		__A							(				nn.Module	):






     def __init__(	self       ,  a__       ,  a__       ,  a__ = "none"						):
       super().__init__()
       _lowerCAmelCase		: Tuple						    =  nn.Linear(a__       ,  a__						)
       _lowerCAmelCase		: int						    =  approximate






     def 					__A						(	self       ,  a__						):
       if gate.device.type != "mps":
         return F.gelu(a__       ,  approximate=self.approximate						)
       # mps: gelu is not implemented for float16
       return F.gelu(gate.to(dtype=torch.floataa						)       ,  approximate=self.approximate						).to(dtype=gate.dtype						)






     def 					__A						(	self       ,  a__						):
       _lowerCAmelCase		: Dict						    =  self.proj(a__						)
       _lowerCAmelCase		: List[Any]						    =  self.gelu(a__						)
       return hidden_states







class 		__A							(				nn.Module	):






     def __init__(	self       ,  a__       ,  a__						):
       super().__init__()
       _lowerCAmelCase		: Dict						    =  nn.Linear(a__       ,  dim_out * 2						)






     def 					__A						(	self       ,  a__						):
       if gate.device.type != "mps":
         return F.gelu(a__						)
       # mps: gelu is not implemented for float16
       return F.gelu(gate.to(dtype=torch.floataa						)						).to(dtype=gate.dtype						)






     def 					__A						(	self       ,  a__						):
       _lowerCAmelCase    ,			_lowerCAmelCase		: Optional[int]						    =  self.proj(a__						).chunk(2       ,  dim=-1						)
       return hidden_states * self.gelu(a__						)







class 		__A							(				nn.Module	):






     def __init__(	self       ,  a__       ,  a__						):
       super().__init__()
       _lowerCAmelCase		: Optional[Any]						    =  nn.Linear(a__       ,  a__						)






     def 					__A						(	self       ,  a__						):
       _lowerCAmelCase		: Dict						    =  self.proj(a__						)
       return x * torch.sigmoid(1.7_0_2 * x						)







class 		__A							(				nn.Module	):






     def __init__(	self       ,  a__       ,  a__						):
       super().__init__()
       _lowerCAmelCase		: Any						    =  nn.Embedding(a__       ,  a__						)
       _lowerCAmelCase		: Tuple						    =  nn.SiLU()
       _lowerCAmelCase		: int						    =  nn.Linear(a__       ,  embedding_dim * 2						)
       _lowerCAmelCase		: Optional[Any]						    =  nn.LayerNorm(a__       ,  elementwise_affine=a__						)






     def 					__A						(	self       ,  a__       ,  a__						):
       _lowerCAmelCase		: Union[str, Any]						    =  self.linear(self.silu(self.emb(a__						)						)						)
       _lowerCAmelCase    ,			_lowerCAmelCase		: Optional[Any]						    =  torch.chunk(a__       ,  2						)
       _lowerCAmelCase		: Dict						    =  self.norm(a__						) * (1 + scale) + shift
       return x







class 		__A							(				nn.Module	):






     def __init__(	self       ,  a__       ,  a__						):
       super().__init__()

       _lowerCAmelCase		: Optional[int]						    =  CombinedTimestepLabelEmbeddings(a__       ,  a__						)

       _lowerCAmelCase		: List[str]						    =  nn.SiLU()
       _lowerCAmelCase		: Union[str, Any]						    =  nn.Linear(a__       ,  6 * embedding_dim       ,  bias=a__						)
       _lowerCAmelCase		: Union[str, Any]						    =  nn.LayerNorm(a__       ,  elementwise_affine=a__       ,  eps=1e-6						)






     def 					__A						(	self       ,  a__       ,  a__       ,  a__       ,  a__=None						):
       _lowerCAmelCase		: Union[str, Any]						    =  self.linear(self.silu(self.emb(a__       ,  a__       ,  hidden_dtype=a__						)						)						)
       _lowerCAmelCase    ,			_lowerCAmelCase    ,			_lowerCAmelCase    ,			_lowerCAmelCase    ,			_lowerCAmelCase    ,			_lowerCAmelCase		: str						    =  emb.chunk(6       ,  dim=1						)
       _lowerCAmelCase		: Union[str, Any]						    =  self.norm(a__						) * (1 + scale_msa[:, None]) + shift_msa[:, None]
       return x, gate_msa, shift_mlp, scale_mlp, gate_mlp







class 		__A							(				nn.Module	):






     def __init__(	self       ,  a__       ,  a__       ,  a__       ,  a__ = None       ,  a__ = 1e-5						):
       super().__init__()
       _lowerCAmelCase		: List[str]						    =  num_groups
       _lowerCAmelCase		: Union[str, Any]						    =  eps

       if act_fn is None:
         _lowerCAmelCase		: Optional[Any]						    =  None
       else:
         _lowerCAmelCase		: Optional[Any]						    =  get_activation(a__						)

       _lowerCAmelCase		: Union[str, Any]						    =  nn.Linear(a__       ,  out_dim * 2						)






     def 					__A						(	self       ,  a__       ,  a__						):
       if self.act:
         _lowerCAmelCase		: Optional[int]						    =  self.act(a__						)
       _lowerCAmelCase		: Union[str, Any]						    =  self.linear(a__						)
       _lowerCAmelCase		: Any						    =  emb[:, :, None, None]
       _lowerCAmelCase    ,			_lowerCAmelCase		: List[Any]						    =  emb.chunk(2       ,  dim=1						)

       _lowerCAmelCase		: str						    =  F.group_norm(a__       ,  self.num_groups       ,  eps=self.eps						)
       _lowerCAmelCase		: Union[str, Any]						    =  x * (1 + scale) + shift
       return x





"""simple docstring"""




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


_a   :	Union[str, Any] 					=     {}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     _a   :	Tuple 					=     ['GPTSw3Tokenizer']


if TYPE_CHECKING:
     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_gpt_swa import GPTSwaTokenizer

else:
     import sys

     _a   :	Optional[int] 					=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




