


from ...configuration_utils import PretrainedConfig
from ...utils import logging


a					  =     logging.get_logger(__name__)

a					  =     {
    'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/config.json',
    'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/config.json'
    # See all FNet models at https://huggingface.co/models?filter=fnet
}







class        UpperCamelCase__   (	__magic_name__   ):
					__SCREAMING_SNAKE_CASE       :  int						=   'fnet'

					def __init__(     self       :   Optional[Any]							,					UpperCamelCase__       :   Dict=32_000							,					UpperCamelCase__       :   List[str]=768							,					UpperCamelCase__       :   Union[str, Any]=12							,					UpperCamelCase__       :   Optional[Any]=3_072							,					UpperCamelCase__       :   str="gelu_new"							,					UpperCamelCase__       :   Optional[int]=0.1							,					UpperCamelCase__       :   Dict=512							,					UpperCamelCase__       :   Dict=4							,					UpperCamelCase__       :   int=0.02							,					UpperCamelCase__       :   Union[str, Any]=1e-12							,					UpperCamelCase__       :   Dict=False							,					UpperCamelCase__       :   Tuple=512							,					UpperCamelCase__       :   Dict=3							,					UpperCamelCase__       :   List[str]=1							,					UpperCamelCase__       :   Tuple=2							,					**UpperCamelCase__       :   Dict							,					):






										'''simple docstring'''






										super().__init__(pad_token_id=UpperCamelCase__							,					bos_token_id=UpperCamelCase__							,					eos_token_id=UpperCamelCase__							,					**UpperCamelCase__       )

										lowercase_  							=			vocab_size
										lowercase_  							=			max_position_embeddings
										lowercase_  							=			hidden_size
										lowercase_  							=			num_hidden_layers
										lowercase_  							=			intermediate_size
										lowercase_  							=			hidden_act
										lowercase_  							=			hidden_dropout_prob
										lowercase_  							=			initializer_range
										lowercase_  							=			type_vocab_size
										lowercase_  							=			layer_norm_eps
										lowercase_  							=			use_tpu_fourier_optimizations
										lowercase_  							=			tpu_short_seq_length






import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin







class        UpperCamelCase__   (	__magic_name__							,				__magic_name__							,				__magic_name__   ):

					@register_to_config
					def __init__(     self       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   float							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   bool = False							,					):






										'''simple docstring'''






										super().__init__()

										lowercase_  							=			nn.Embedding(UpperCamelCase__							,					UpperCamelCase__       )

										lowercase_  							=			nn.Embedding(UpperCamelCase__							,					UpperCamelCase__       )
										lowercase_  							=			False

										lowercase_  							=			nn.Dropout(p=UpperCamelCase__       )

										lowercase_  							=			TaConfig(
										    vocab_size=UpperCamelCase__							,					d_model=UpperCamelCase__							,					num_heads=UpperCamelCase__							,					d_kv=UpperCamelCase__							,					d_ff=UpperCamelCase__							,					dropout_rate=UpperCamelCase__							,					feed_forward_proj=UpperCamelCase__							,					is_decoder=UpperCamelCase__							,					is_encoder_decoder=UpperCamelCase__							,					)

										lowercase_  							=			nn.ModuleList()
										for lyr_num in range(UpperCamelCase__       ):
															lowercase_  							=			TaBlock(UpperCamelCase__       )
															self.encoders.append(UpperCamelCase__       )

										lowercase_  							=			TaLayerNorm(UpperCamelCase__       )
										lowercase_  							=			nn.Dropout(p=UpperCamelCase__       )







					def   UpperCAmelCase__       (     self       :   Optional[Any]							,					UpperCamelCase__       :   Optional[int]							,					UpperCamelCase__       :   str       ):






										'''simple docstring'''






										lowercase_  							=			self.token_embedder(UpperCamelCase__       )

										lowercase_  							=			encoder_input_tokens.shape[1]
										lowercase_  							=			torch.arange(UpperCamelCase__							,					device=encoder_input_tokens.device       )
										x += self.position_encoding(UpperCamelCase__       )

										lowercase_  							=			self.dropout_pre(UpperCamelCase__       )

										# inverted the attention mask
										lowercase_  							=			encoder_input_tokens.size()
										lowercase_  							=			self.get_extended_attention_mask(UpperCamelCase__							,					UpperCamelCase__       )

										for lyr in self.encoders:
															lowercase_  							=			lyr(UpperCamelCase__							,					UpperCamelCase__       )[0]
										lowercase_  							=			self.layer_norm(UpperCamelCase__       )

										return self.dropout_post(UpperCamelCase__       ), encoder_inputs_mask


