


from ...configuration_utils import PretrainedConfig
from ...utils import logging


a					  =     logging.get_logger(__name__)

a					  =     {
    'facebook/s2t-wav2vec2-large-en-de': (
        'https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/config.json'
    ),
    # See all Speech2Text models at https://huggingface.co/models?filter=speech2text2
}







class        UpperCamelCase__   (	__magic_name__   ):
					__SCREAMING_SNAKE_CASE       :  str						=   'speech_to_text_2'
					__SCREAMING_SNAKE_CASE       :  str						=   ['past_key_values']
					__SCREAMING_SNAKE_CASE       :  Dict						=   {'num_attention_heads': 'decoder_attention_heads', 'hidden_size': 'd_model'}

					def __init__(     self       :   Optional[Any]							,					UpperCamelCase__       :   Union[str, Any]=10_000							,					UpperCamelCase__       :   List[str]=6							,					UpperCamelCase__       :   Optional[int]=2_048							,					UpperCamelCase__       :   int=4							,					UpperCamelCase__       :   List[Any]=0.0							,					UpperCamelCase__       :   int=True							,					UpperCamelCase__       :   str="relu"							,					UpperCamelCase__       :   int=256							,					UpperCamelCase__       :   Union[str, Any]=0.1							,					UpperCamelCase__       :   Any=0.0							,					UpperCamelCase__       :   Optional[int]=0.0							,					UpperCamelCase__       :   Optional[Any]=0.02							,					UpperCamelCase__       :   List[Any]=2							,					UpperCamelCase__       :   List[Any]=True							,					UpperCamelCase__       :   Optional[Any]=1							,					UpperCamelCase__       :   List[Any]=0							,					UpperCamelCase__       :   Any=2							,					UpperCamelCase__       :   Union[str, Any]=1_024							,					**UpperCamelCase__       :   str							,					):






										'''simple docstring'''






										lowercase_  							=			vocab_size
										lowercase_  							=			d_model
										lowercase_  							=			decoder_ffn_dim
										lowercase_  							=			decoder_layers
										lowercase_  							=			decoder_attention_heads
										lowercase_  							=			dropout
										lowercase_  							=			attention_dropout
										lowercase_  							=			activation_dropout
										lowercase_  							=			activation_function
										lowercase_  							=			init_std
										lowercase_  							=			decoder_layerdrop
										lowercase_  							=			use_cache
										lowercase_  							=			decoder_layers
										lowercase_  							=			scale_embedding  # scale factor will be sqrt(d_model) if True
										lowercase_  							=			max_target_positions

										super().__init__(
										    pad_token_id=UpperCamelCase__							,					bos_token_id=UpperCamelCase__							,					eos_token_id=UpperCamelCase__							,					decoder_start_token_id=UpperCamelCase__							,					**UpperCamelCase__							,					)






import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin







class        UpperCamelCase__   (	__magic_name__							,				__magic_name__							,				__magic_name__   ):

					@register_to_config
					def __init__(     self       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   float							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   bool = False							,					):






										'''simple docstring'''






										super().__init__()

										lowercase_  							=			nn.Embedding(UpperCamelCase__							,					UpperCamelCase__       )

										lowercase_  							=			nn.Embedding(UpperCamelCase__							,					UpperCamelCase__       )
										lowercase_  							=			False

										lowercase_  							=			nn.Dropout(p=UpperCamelCase__       )

										lowercase_  							=			TaConfig(
										    vocab_size=UpperCamelCase__							,					d_model=UpperCamelCase__							,					num_heads=UpperCamelCase__							,					d_kv=UpperCamelCase__							,					d_ff=UpperCamelCase__							,					dropout_rate=UpperCamelCase__							,					feed_forward_proj=UpperCamelCase__							,					is_decoder=UpperCamelCase__							,					is_encoder_decoder=UpperCamelCase__							,					)

										lowercase_  							=			nn.ModuleList()
										for lyr_num in range(UpperCamelCase__       ):
															lowercase_  							=			TaBlock(UpperCamelCase__       )
															self.encoders.append(UpperCamelCase__       )

										lowercase_  							=			TaLayerNorm(UpperCamelCase__       )
										lowercase_  							=			nn.Dropout(p=UpperCamelCase__       )







					def   UpperCAmelCase__       (     self       :   Optional[Any]							,					UpperCamelCase__       :   Optional[int]							,					UpperCamelCase__       :   str       ):






										'''simple docstring'''






										lowercase_  							=			self.token_embedder(UpperCamelCase__       )

										lowercase_  							=			encoder_input_tokens.shape[1]
										lowercase_  							=			torch.arange(UpperCamelCase__							,					device=encoder_input_tokens.device       )
										x += self.position_encoding(UpperCamelCase__       )

										lowercase_  							=			self.dropout_pre(UpperCamelCase__       )

										# inverted the attention mask
										lowercase_  							=			encoder_input_tokens.size()
										lowercase_  							=			self.get_extended_attention_mask(UpperCamelCase__							,					UpperCamelCase__       )

										for lyr in self.encoders:
															lowercase_  							=			lyr(UpperCamelCase__							,					UpperCamelCase__       )[0]
										lowercase_  							=			self.layer_norm(UpperCamelCase__       )

										return self.dropout_post(UpperCamelCase__       ), encoder_inputs_mask


