


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a					  =     {
    'configuration_electra': ['ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ElectraConfig', 'ElectraOnnxConfig'],
    'tokenization_electra': ['ElectraTokenizer'],
}

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							a					  =     ['ElectraTokenizerFast']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							a					  =     [
							    'ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'ElectraForCausalLM',
							    'ElectraForMaskedLM',
							    'ElectraForMultipleChoice',
							    'ElectraForPreTraining',
							    'ElectraForQuestionAnswering',
							    'ElectraForSequenceClassification',
							    'ElectraForTokenClassification',
							    'ElectraModel',
							    'ElectraPreTrainedModel',
							    'load_tf_weights_in_electra',
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							a					  =     [
							    'TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'TFElectraForMaskedLM',
							    'TFElectraForMultipleChoice',
							    'TFElectraForPreTraining',
							    'TFElectraForQuestionAnswering',
							    'TFElectraForSequenceClassification',
							    'TFElectraForTokenClassification',
							    'TFElectraModel',
							    'TFElectraPreTrainedModel',
							]

try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							a					  =     [
							    'FlaxElectraForCausalLM',
							    'FlaxElectraForMaskedLM',
							    'FlaxElectraForMultipleChoice',
							    'FlaxElectraForPreTraining',
							    'FlaxElectraForQuestionAnswering',
							    'FlaxElectraForSequenceClassification',
							    'FlaxElectraForTokenClassification',
							    'FlaxElectraModel',
							    'FlaxElectraPreTrainedModel',
							]


if TYPE_CHECKING:
							from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig, ElectraOnnxConfig
							from .tokenization_electra import ElectraTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_electra_fast import ElectraTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_electra import (
														    ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
														    ElectraForCausalLM,
														    ElectraForMaskedLM,
														    ElectraForMultipleChoice,
														    ElectraForPreTraining,
														    ElectraForQuestionAnswering,
														    ElectraForSequenceClassification,
														    ElectraForTokenClassification,
														    ElectraModel,
														    ElectraPreTrainedModel,
														    load_tf_weights_in_electra,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_electra import (
														    TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFElectraForMaskedLM,
														    TFElectraForMultipleChoice,
														    TFElectraForPreTraining,
														    TFElectraForQuestionAnswering,
														    TFElectraForSequenceClassification,
														    TFElectraForTokenClassification,
														    TFElectraModel,
														    TFElectraPreTrainedModel,
														)

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_electra import (
														    FlaxElectraForCausalLM,
														    FlaxElectraForMaskedLM,
														    FlaxElectraForMultipleChoice,
														    FlaxElectraForPreTraining,
														    FlaxElectraForQuestionAnswering,
														    FlaxElectraForSequenceClassification,
														    FlaxElectraForTokenClassification,
														    FlaxElectraModel,
														    FlaxElectraPreTrainedModel,
														)

else:
							import sys

							a					  =     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin







class        UpperCamelCase__   (	__magic_name__							,				__magic_name__							,				__magic_name__   ):

					@register_to_config
					def __init__(     self       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   float							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   bool = False							,					):






										'''simple docstring'''






										super().__init__()

										lowercase_  							=			nn.Embedding(UpperCamelCase__							,					UpperCamelCase__       )

										lowercase_  							=			nn.Embedding(UpperCamelCase__							,					UpperCamelCase__       )
										lowercase_  							=			False

										lowercase_  							=			nn.Dropout(p=UpperCamelCase__       )

										lowercase_  							=			TaConfig(
										    vocab_size=UpperCamelCase__							,					d_model=UpperCamelCase__							,					num_heads=UpperCamelCase__							,					d_kv=UpperCamelCase__							,					d_ff=UpperCamelCase__							,					dropout_rate=UpperCamelCase__							,					feed_forward_proj=UpperCamelCase__							,					is_decoder=UpperCamelCase__							,					is_encoder_decoder=UpperCamelCase__							,					)

										lowercase_  							=			nn.ModuleList()
										for lyr_num in range(UpperCamelCase__       ):
															lowercase_  							=			TaBlock(UpperCamelCase__       )
															self.encoders.append(UpperCamelCase__       )

										lowercase_  							=			TaLayerNorm(UpperCamelCase__       )
										lowercase_  							=			nn.Dropout(p=UpperCamelCase__       )







					def   UpperCAmelCase__       (     self       :   Optional[Any]							,					UpperCamelCase__       :   Optional[int]							,					UpperCamelCase__       :   str       ):






										'''simple docstring'''






										lowercase_  							=			self.token_embedder(UpperCamelCase__       )

										lowercase_  							=			encoder_input_tokens.shape[1]
										lowercase_  							=			torch.arange(UpperCamelCase__							,					device=encoder_input_tokens.device       )
										x += self.position_encoding(UpperCamelCase__       )

										lowercase_  							=			self.dropout_pre(UpperCamelCase__       )

										# inverted the attention mask
										lowercase_  							=			encoder_input_tokens.size()
										lowercase_  							=			self.get_extended_attention_mask(UpperCamelCase__							,					UpperCamelCase__       )

										for lyr in self.encoders:
															lowercase_  							=			lyr(UpperCamelCase__							,					UpperCamelCase__       )[0]
										lowercase_  							=			self.layer_norm(UpperCamelCase__       )

										return self.dropout_post(UpperCamelCase__       ), encoder_inputs_mask


