



import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin

class   _A   ( _lowercase  ,						_lowercase	):


						'''simple docstring'''





						@register_to_config
						def __init__(	self	:							Dict							,      lowerCamelCase	:							int = 128							,      lowerCamelCase	:							int = 256							,      lowerCamelCase	:							float = 2000.0							,      lowerCamelCase	:							int = 768							,      lowerCamelCase	:							int = 12							,      lowerCamelCase	:							int = 12							,      lowerCamelCase	:							int = 64							,      lowerCamelCase	:							int = 2_048							,      lowerCamelCase	:							float = 0.1							,      ):






							'''simple docstring'''



							super().__init__()

							__lowercase		       =			nn.Sequential(
							    nn.Linear(lowerCamelCase							,      d_model * 4							,      bias=lowerCamelCase	)							,      nn.SiLU()							,      nn.Linear(d_model * 4							,      d_model * 4							,      bias=lowerCamelCase	)							,      nn.SiLU()							,      )

							__lowercase		       =			nn.Embedding(lowerCamelCase							,      lowerCamelCase	)
							__lowercase		       =			False

							__lowercase		       =			nn.Linear(lowerCamelCase							,      lowerCamelCase							,      bias=lowerCamelCase	)

							__lowercase		       =			nn.Dropout(p=lowerCamelCase	)

							__lowercase		       =			nn.ModuleList()
							for lyr_num in range(lowerCamelCase	):
								# FiLM conditional T5 decoder
								__lowercase		       =			DecoderLayer(d_model=lowerCamelCase							,      d_kv=lowerCamelCase							,      num_heads=lowerCamelCase							,      d_ff=lowerCamelCase							,      dropout_rate=lowerCamelCase	)
								self.decoders.append(lowerCamelCase	)

							__lowercase		       =			TaLayerNorm(lowerCamelCase	)

							__lowercase		       =			nn.Dropout(p=lowerCamelCase	)
							__lowercase		       =			nn.Linear(lowerCamelCase							,      lowerCamelCase							,      bias=lowerCamelCase	)



						def     _snake_case	(	self	:							Union[str, Any]							,      lowerCamelCase	:							Optional[Any]							,      lowerCamelCase	:							Optional[int]	):






							'''simple docstring'''



							__lowercase		       =			torch.mul(query_input.unsqueeze(-1	)							,      key_input.unsqueeze(-2	)	)
							return mask.unsqueeze(-3	)



						def     _snake_case	(	self	:							List[Any]							,      lowerCamelCase	:							Any							,      lowerCamelCase	:							List[Any]							,      lowerCamelCase	:							str	):






							'''simple docstring'''



							__lowercase       ,    __lowercase       ,    __lowercase		       =			decoder_input_tokens.shape
							assert decoder_noise_time.shape == (batch,)

							# decoder_noise_time is in [0, 1), so rescale to expected timing range.
							__lowercase		       =			get_timestep_embedding(
							    decoder_noise_time * self.config.max_decoder_noise_time							,      embedding_dim=self.config.d_model							,      max_period=self.config.max_decoder_noise_time							,      ).to(dtype=self.dtype	)

							__lowercase		       =			self.conditioning_emb(lowerCamelCase	).unsqueeze(1	)

							assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

							__lowercase		       =			decoder_input_tokens.shape[1]

							# If we want to use relative positions for audio context, we can just offset
							# this sequence by the length of encodings_and_masks.
							__lowercase		       =			torch.broadcast_to(
							    torch.arange(lowerCamelCase							,      device=decoder_input_tokens.device	)							,      (batch, seq_length)							,      )

							__lowercase		       =			self.position_encoding(lowerCamelCase	)

							__lowercase		       =			self.continuous_inputs_projection(lowerCamelCase	)
							inputs += position_encodings
							__lowercase		       =			self.dropout(lowerCamelCase	)

							# decoder: No padding present.
							__lowercase		       =			torch.ones(
							    decoder_input_tokens.shape[:2]							,      device=decoder_input_tokens.device							,      dtype=inputs.dtype	)

							# Translate encoding masks to encoder-decoder masks.
							__lowercase		       =			[(x, self.encoder_decoder_mask(lowerCamelCase							,      lowerCamelCase	)) for x, y in encodings_and_masks]

							# cross attend style: concat encodings
							__lowercase		       =			torch.cat([x[0] for x in encodings_and_encdec_masks]							,      dim=1	)
							__lowercase		       =			torch.cat([x[1] for x in encodings_and_encdec_masks]							,      dim=-1	)

							for lyr in self.decoders:
								__lowercase		       =			lyr(
								    lowerCamelCase							,      conditioning_emb=lowerCamelCase							,      encoder_hidden_states=lowerCamelCase							,      encoder_attention_mask=lowerCamelCase							,      )[0]

							__lowercase		       =			self.decoder_norm(lowerCamelCase	)
							__lowercase		       =			self.post_dropout(lowerCamelCase	)

							__lowercase		       =			self.spec_out(lowerCamelCase	)
							return spec_out

class   _A   ( nn.Module	):


						'''simple docstring'''





						def __init__(	self	:							Tuple							,      lowerCamelCase	:							Optional[int]							,      lowerCamelCase	:							List[str]							,      lowerCamelCase	:							int							,      lowerCamelCase	:							str							,      lowerCamelCase	:							Optional[Any]							,      lowerCamelCase	:							Union[str, Any]=1e-6	):






							'''simple docstring'''



							super().__init__()
							__lowercase		       =			nn.ModuleList()

							# cond self attention: layer 0
							self.layer.append(
							    TaLayerSelfAttentionCond(d_model=lowerCamelCase							,      d_kv=lowerCamelCase							,      num_heads=lowerCamelCase							,      dropout_rate=lowerCamelCase	)	)

							# cross attention: layer 1
							self.layer.append(
							    TaLayerCrossAttention(
							        d_model=lowerCamelCase							,      d_kv=lowerCamelCase							,      num_heads=lowerCamelCase							,      dropout_rate=lowerCamelCase							,      layer_norm_epsilon=lowerCamelCase							,      )	)

							# Film Cond MLP + dropout: last layer
							self.layer.append(
							    TaLayerFFCond(d_model=lowerCamelCase							,      d_ff=lowerCamelCase							,      dropout_rate=lowerCamelCase							,      layer_norm_epsilon=lowerCamelCase	)	)



						def     _snake_case	(	self	:							Union[str, Any]							,      lowerCamelCase	:							Optional[Any]							,      lowerCamelCase	:							Any=None							,      lowerCamelCase	:							List[str]=None							,      lowerCamelCase	:							List[Any]=None							,      lowerCamelCase	:							int=None							,      lowerCamelCase	:							List[str]=None							,      ):






							'''simple docstring'''



							__lowercase		       =			self.layer[0](
							    lowerCamelCase							,      conditioning_emb=lowerCamelCase							,      attention_mask=lowerCamelCase							,      )

							if encoder_hidden_states is not None:
								__lowercase		       =			torch.where(encoder_attention_mask > 0							,      0							,      -1e10	).to(
								    encoder_hidden_states.dtype	)

								__lowercase		       =			self.layer[1](
								    lowerCamelCase							,      key_value_states=lowerCamelCase							,      attention_mask=lowerCamelCase							,      )

							# Apply Film Conditional Feed Forward layer
							__lowercase		       =			self.layer[-1](lowerCamelCase							,      lowerCamelCase	)

							return (hidden_states,)

class   _A   ( nn.Module	):


						'''simple docstring'''





						def __init__(	self	:							Any							,      lowerCamelCase	:							Tuple							,      lowerCamelCase	:							Union[str, Any]							,      lowerCamelCase	:							List[str]							,      lowerCamelCase	:							int	):






							'''simple docstring'''



							super().__init__()
							__lowercase		       =			TaLayerNorm(lowerCamelCase	)
							__lowercase		       =			TaFiLMLayer(in_features=d_model * 4							,      out_features=lowerCamelCase	)
							__lowercase		       =			Attention(query_dim=lowerCamelCase							,      heads=lowerCamelCase							,      dim_head=lowerCamelCase							,      out_bias=lowerCamelCase							,      scale_qk=lowerCamelCase	)
							__lowercase		       =			nn.Dropout(lowerCamelCase	)



						def     _snake_case	(	self	:							Optional[Any]							,      lowerCamelCase	:							Tuple							,      lowerCamelCase	:							List[str]=None							,      lowerCamelCase	:							Optional[Any]=None							,      ):






							'''simple docstring'''



							__lowercase		       =			self.layer_norm(lowerCamelCase	)

							if conditioning_emb is not None:
								__lowercase		       =			self.FiLMLayer(lowerCamelCase							,      lowerCamelCase	)

							# Self-attention block
							__lowercase		       =			self.attention(lowerCamelCase	)

							__lowercase		       =			hidden_states + self.dropout(lowerCamelCase	)

							return hidden_states

class   _A   ( nn.Module	):


						'''simple docstring'''





						def __init__(	self	:							Optional[Any]							,      lowerCamelCase	:							Optional[int]							,      lowerCamelCase	:							List[str]							,      lowerCamelCase	:							Any							,      lowerCamelCase	:							List[str]							,      lowerCamelCase	:							Any	):






							'''simple docstring'''



							super().__init__()
							__lowercase		       =			Attention(query_dim=lowerCamelCase							,      heads=lowerCamelCase							,      dim_head=lowerCamelCase							,      out_bias=lowerCamelCase							,      scale_qk=lowerCamelCase	)
							__lowercase		       =			TaLayerNorm(lowerCamelCase							,      eps=lowerCamelCase	)
							__lowercase		       =			nn.Dropout(lowerCamelCase	)



						def     _snake_case	(	self	:							Dict							,      lowerCamelCase	:							Union[str, Any]							,      lowerCamelCase	:							Dict=None							,      lowerCamelCase	:							str=None							,      ):






							'''simple docstring'''



							__lowercase		       =			self.layer_norm(lowerCamelCase	)
							__lowercase		       =			self.attention(
							    lowerCamelCase							,      encoder_hidden_states=lowerCamelCase							,      attention_mask=attention_mask.squeeze(1	)							,      )
							__lowercase		       =			hidden_states + self.dropout(lowerCamelCase	)
							return layer_output

class   _A   ( nn.Module	):


						'''simple docstring'''





						def __init__(	self	:							Tuple							,      lowerCamelCase	:							int							,      lowerCamelCase	:							List[str]							,      lowerCamelCase	:							str							,      lowerCamelCase	:							Dict	):






							'''simple docstring'''



							super().__init__()
							__lowercase		       =			TaDenseGatedActDense(d_model=lowerCamelCase							,      d_ff=lowerCamelCase							,      dropout_rate=lowerCamelCase	)
							__lowercase		       =			TaFiLMLayer(in_features=d_model * 4							,      out_features=lowerCamelCase	)
							__lowercase		       =			TaLayerNorm(lowerCamelCase							,      eps=lowerCamelCase	)
							__lowercase		       =			nn.Dropout(lowerCamelCase	)



						def     _snake_case	(	self	:							str							,      lowerCamelCase	:							Tuple							,      lowerCamelCase	:							Dict=None	):






							'''simple docstring'''



							__lowercase		       =			self.layer_norm(lowerCamelCase	)
							if conditioning_emb is not None:
								__lowercase		       =			self.film(lowerCamelCase							,      lowerCamelCase	)

							__lowercase		       =			self.DenseReluDense(lowerCamelCase	)
							__lowercase		       =			hidden_states + self.dropout(lowerCamelCase	)
							return hidden_states

class   _A   ( nn.Module	):


						'''simple docstring'''





						def __init__(	self	:							Dict							,      lowerCamelCase	:							Tuple							,      lowerCamelCase	:							Tuple							,      lowerCamelCase	:							Optional[Any]	):






							'''simple docstring'''



							super().__init__()
							__lowercase		       =			nn.Linear(lowerCamelCase							,      lowerCamelCase							,      bias=lowerCamelCase	)
							__lowercase		       =			nn.Linear(lowerCamelCase							,      lowerCamelCase							,      bias=lowerCamelCase	)
							__lowercase		       =			nn.Linear(lowerCamelCase							,      lowerCamelCase							,      bias=lowerCamelCase	)
							__lowercase		       =			nn.Dropout(lowerCamelCase	)
							__lowercase		       =			NewGELUActivation()



						def     _snake_case	(	self	:							Optional[int]							,      lowerCamelCase	:							Any	):






							'''simple docstring'''



							__lowercase		       =			self.act(self.wi_a(lowerCamelCase	)	)
							__lowercase		       =			self.wi_a(lowerCamelCase	)
							__lowercase		       =			hidden_gelu * hidden_linear
							__lowercase		       =			self.dropout(lowerCamelCase	)

							__lowercase		       =			self.wo(lowerCamelCase	)
							return hidden_states

class   _A   ( nn.Module	):


						'''simple docstring'''





						def __init__(	self	:							Any							,      lowerCamelCase	:							str							,      lowerCamelCase	:							Optional[int]=1e-6	):






							'''simple docstring'''



							super().__init__()
							__lowercase		       =			nn.Parameter(torch.ones(lowerCamelCase	)	)
							__lowercase		       =			eps



						def     _snake_case	(	self	:							Dict							,      lowerCamelCase	:							str	):






							'''simple docstring'''



							__lowercase		       =			hidden_states.to(torch.floataa	).pow(2	).mean(-1							,      keepdim=lowerCamelCase	)
							__lowercase		       =			hidden_states * torch.rsqrt(variance + self.variance_epsilon	)

							# convert into half-precision if necessary
							if self.weight.dtype in [torch.floataa, torch.bfloataa]:
								__lowercase		       =			hidden_states.to(self.weight.dtype	)

							return self.weight * hidden_states

class   _A   ( nn.Module	):


						'''simple docstring'''





						def     _snake_case	(	self	:							List[str]							,      lowerCamelCase	:							torch.Tensor	):






							'''simple docstring'''



							return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi	) * (input + 0.04_4715 * torch.pow(lowerCamelCase							,      3.0	))	))

class   _A   ( nn.Module	):


						'''simple docstring'''





						def __init__(	self	:							Union[str, Any]							,      lowerCamelCase	:							Optional[Any]							,      lowerCamelCase	:							Optional[int]	):






							'''simple docstring'''



							super().__init__()
							__lowercase		       =			nn.Linear(lowerCamelCase							,      out_features * 2							,      bias=lowerCamelCase	)



						def     _snake_case	(	self	:							List[Any]							,      lowerCamelCase	:							List[str]							,      lowerCamelCase	:							Optional[int]	):






							'''simple docstring'''



							__lowercase		       =			self.scale_bias(lowerCamelCase	)
							__lowercase       ,    __lowercase		       =			torch.chunk(lowerCamelCase							,      2							,      -1	)
							__lowercase		       =			x * (1 + scale) + shift
							return x








# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


snake_case__					:     List[str]    		=   {
    """configuration_mgp_str""": ["""MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MgpstrConfig"""],
    """processing_mgp_str""": ["""MgpstrProcessor"""],
    """tokenization_mgp_str""": ["""MgpstrTokenizer"""],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	snake_case__					:     Dict    		=   [
	    """MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """MgpstrModel""",
	    """MgpstrPreTrainedModel""",
	    """MgpstrForSceneTextRecognition""",
	]

if TYPE_CHECKING:
	from .configuration_mgp_str import MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP, MgpstrConfig
	from .processing_mgp_str import MgpstrProcessor
	from .tokenization_mgp_str import MgpstrTokenizer

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_mgp_str import (
		    MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST,
		    MgpstrForSceneTextRecognition,
		    MgpstrModel,
		    MgpstrPreTrainedModel,
		)
else:
	import sys

	snake_case__					:     Union[str, Any]    		=   _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



