





from typing import Optional, Tuple, Union

import tensorflow as tf

from ...activations_tf import ACTaFN
from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
from ...modeling_tf_outputs import (
    TFBaseModelOutputWithNoAttention,
    TFBaseModelOutputWithPoolingAndNoAttention,
    TFSequenceClassifierOutput,
)
from ...modeling_tf_utils import TFPreTrainedModel, TFSequenceClassificationLoss, keras_serializable, unpack_inputs
from ...tf_utils import shape_list
from ...utils import logging
from .configuration_regnet import RegNetConfig


lowercase_		:				Dict				=						logging.get_logger(__name__)

# General docstring
lowercase_		:				Optional[int]				=						'''RegNetConfig'''

# Base docstring
lowercase_		:				List[Any]				=						'''facebook/regnet-y-040'''
lowercase_		:				int				=						[1, 1088, 7, 7]

# Image classification docstring
lowercase_		:				Optional[int]				=						'''facebook/regnet-y-040'''
lowercase_		:				List[str]				=						'''tabby, tabby cat'''

lowercase_		:				int				=						[
    '''facebook/regnet-y-040''',
    # See all regnet models at https://huggingface.co/models?filter=regnet
]


class 			lowercase				(							tf.keras.layers.Layer     ):



						"""simple docstring"""

						def __init__(    self			:		int ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int = 3 ,				lowerCamelCase_			:		int = 1 ,				lowerCamelCase_			:		int = 1 ,				lowerCamelCase_			:		Optional[str] = "relu" ,				**lowerCamelCase_			:		Dict ,				):





										'''simple docstring'''







										super().__init__(**lowerCamelCase_					)
										# The padding and conv has been verified in
										# https://colab.research.google.com/gist/sayakpaul/854bc10eeaf21c9ee2119e0b9f3841a7/scratchpad.ipynb
										_snake_case :			List[Any]  						=   tf.keras.layers.ZeroPaddingaD(padding=kernel_size // 2					)
										_snake_case :			Tuple  						=   tf.keras.layers.ConvaD(
										    filters=lowerCamelCase_ ,				kernel_size=lowerCamelCase_ ,				strides=lowerCamelCase_ ,				padding='VALID' ,				groups=lowerCamelCase_ ,				use_bias=lowerCamelCase_ ,				name='convolution' ,				)
										_snake_case :			Optional[Any]  						=   tf.keras.layers.BatchNormalization(epsilon=1e-5 ,				momentum=0.9 ,				name='normalization'					)
										_snake_case :			Dict  						=   ACTaFN[activation] if activation is not None else tf.identity







						def 		__UpperCAmelCase       (    self			:		str ,				lowerCamelCase_			:		Optional[Any]					):





										'''simple docstring'''







										_snake_case :			Any  						=   self.convolution(self.padding(lowerCamelCase_					)					)
										_snake_case :			Optional[int]  						=   self.normalization(lowerCamelCase_					)
										_snake_case :			Tuple  						=   self.activation(lowerCamelCase_					)
										return hidden_state



class 			lowercase				(							tf.keras.layers.Layer     ):



						"""simple docstring"""

						def __init__(    self			:		str ,				lowerCamelCase_			:		RegNetConfig ,				**lowerCamelCase_			:		List[Any]					):





										'''simple docstring'''







										super().__init__(**lowerCamelCase_					)
										_snake_case :			str  						=   config.num_channels
										_snake_case :			Tuple  						=   TFRegNetConvLayer(
										    out_channels=config.embedding_size ,				kernel_size=3 ,				stride=2 ,				activation=config.hidden_act ,				name='embedder' ,				)







						def 		__UpperCAmelCase       (    self			:		int ,				lowerCamelCase_			:		Optional[Any]					):





										'''simple docstring'''







										_snake_case :			int  						=   shape_list(lowerCamelCase_					)[1]
										if tf.executing_eagerly() and num_channels != self.num_channels:
														raise ValueError(
														    'Make sure that the channel dimension of the pixel values match with the one set in the configuration.'					)

										# When running on CPU, `tf.keras.layers.Conv2D` doesn't support `NCHW` format.
										# So change the input format from `NCHW` to `NHWC`.
										# shape = (batch_size, in_height, in_width, in_channels=num_channels)
										_snake_case :			int  						=   tf.transpose(lowerCamelCase_ ,				perm=(0, 2, 3, 1)					)
										_snake_case :			Optional[int]  						=   self.embedder(lowerCamelCase_					)
										return hidden_state



class 			lowercase				(							tf.keras.layers.Layer     ):



						"""simple docstring"""

						def __init__(    self			:		int ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int = 2 ,				**lowerCamelCase_			:		Dict					):





										'''simple docstring'''







										super().__init__(**lowerCamelCase_					)
										_snake_case :			int  						=   tf.keras.layers.ConvaD(
										    filters=lowerCamelCase_ ,				kernel_size=1 ,				strides=lowerCamelCase_ ,				use_bias=lowerCamelCase_ ,				name='convolution'					)
										_snake_case :			Tuple  						=   tf.keras.layers.BatchNormalization(epsilon=1e-5 ,				momentum=0.9 ,				name='normalization'					)







						def 		__UpperCAmelCase       (    self			:		Tuple ,				lowerCamelCase_			:		tf.Tensor ,				lowerCamelCase_			:		bool = False					):





										'''simple docstring'''







										return self.normalization(self.convolution(lowerCamelCase_					) ,				training=lowerCamelCase_					)



class 			lowercase				(							tf.keras.layers.Layer     ):



						"""simple docstring"""

						def __init__(    self			:		List[str] ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int ,				**lowerCamelCase_			:		List[Any]					):





										'''simple docstring'''







										super().__init__(**lowerCamelCase_					)
										_snake_case :			Optional[int]  						=   tf.keras.layers.GlobalAveragePoolingaD(keepdims=lowerCamelCase_ ,				name='pooler'					)
										_snake_case :			int  						=   [
										    tf.keras.layers.ConvaD(filters=lowerCamelCase_ ,				kernel_size=1 ,				activation='relu' ,				name='attention.0'					),
										    tf.keras.layers.ConvaD(filters=lowerCamelCase_ ,				kernel_size=1 ,				activation='sigmoid' ,				name='attention.2'					),
										]







						def 		__UpperCAmelCase       (    self			:		int ,				lowerCamelCase_			:		Dict					):





										'''simple docstring'''







										_snake_case :			int  						=   self.pooler(lowerCamelCase_					)
										for layer_module in self.attention:
														_snake_case :			Dict  						=   layer_module(lowerCamelCase_					)
										_snake_case :			Tuple  						=   hidden_state * pooled
										return hidden_state



class 			lowercase				(							tf.keras.layers.Layer     ):



						"""simple docstring"""

						def __init__(    self			:		List[str] ,				lowerCamelCase_			:		RegNetConfig ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int = 1 ,				**lowerCamelCase_			:		Optional[Any]					):





										'''simple docstring'''







										super().__init__(**lowerCamelCase_					)
										_snake_case :			List[Any]  						=   in_channels != out_channels or stride != 1
										_snake_case :			Tuple  						=   max(1 ,				out_channels // config.groups_width					)
										_snake_case :			List[str]  						=   (
										    TFRegNetShortCut(lowerCamelCase_ ,				stride=lowerCamelCase_ ,				name='shortcut'					)
										    if should_apply_shortcut
										    else tf.keras.layers.Activation('linear' ,				name='shortcut'					)
										)
										# `self.layers` instead of `self.layer` because that is a reserved argument.
										_snake_case :			Optional[int]  						=   [
										    TFRegNetConvLayer(lowerCamelCase_ ,				kernel_size=1 ,				activation=config.hidden_act ,				name='layer.0'					),
										    TFRegNetConvLayer(
										        lowerCamelCase_ ,				stride=lowerCamelCase_ ,				groups=lowerCamelCase_ ,				activation=config.hidden_act ,				name='layer.1'					),
										    TFRegNetConvLayer(lowerCamelCase_ ,				kernel_size=1 ,				activation=lowerCamelCase_ ,				name='layer.2'					),
										]
										_snake_case :			int  						=   ACTaFN[config.hidden_act]







						def 		__UpperCAmelCase       (    self			:		int ,				lowerCamelCase_			:		int					):





										'''simple docstring'''







										_snake_case :			Dict  						=   hidden_state
										for layer_module in self.layers:
														_snake_case :			Union[str, Any]  						=   layer_module(lowerCamelCase_					)
										_snake_case :			Optional[Any]  						=   self.shortcut(lowerCamelCase_					)
										hidden_state += residual
										_snake_case :			Union[str, Any]  						=   self.activation(lowerCamelCase_					)
										return hidden_state



class 			lowercase				(							tf.keras.layers.Layer     ):



						"""simple docstring"""

						def __init__(    self			:		Optional[int] ,				lowerCamelCase_			:		RegNetConfig ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int = 1 ,				**lowerCamelCase_			:		Optional[Any]					):





										'''simple docstring'''







										super().__init__(**lowerCamelCase_					)
										_snake_case :			List[str]  						=   in_channels != out_channels or stride != 1
										_snake_case :			Optional[int]  						=   max(1 ,				out_channels // config.groups_width					)
										_snake_case :			Optional[Any]  						=   (
										    TFRegNetShortCut(lowerCamelCase_ ,				stride=lowerCamelCase_ ,				name='shortcut'					)
										    if should_apply_shortcut
										    else tf.keras.layers.Activation('linear' ,				name='shortcut'					)
										)
										_snake_case :			Union[str, Any]  						=   [
										    TFRegNetConvLayer(lowerCamelCase_ ,				kernel_size=1 ,				activation=config.hidden_act ,				name='layer.0'					),
										    TFRegNetConvLayer(
										        lowerCamelCase_ ,				stride=lowerCamelCase_ ,				groups=lowerCamelCase_ ,				activation=config.hidden_act ,				name='layer.1'					),
										    TFRegNetSELayer(lowerCamelCase_ ,				reduced_channels=int(round(in_channels / 4					)					) ,				name='layer.2'					),
										    TFRegNetConvLayer(lowerCamelCase_ ,				kernel_size=1 ,				activation=lowerCamelCase_ ,				name='layer.3'					),
										]
										_snake_case :			List[Any]  						=   ACTaFN[config.hidden_act]







						def 		__UpperCAmelCase       (    self			:		str ,				lowerCamelCase_			:		Optional[int]					):





										'''simple docstring'''







										_snake_case :			Optional[int]  						=   hidden_state
										for layer_module in self.layers:
														_snake_case :			str  						=   layer_module(lowerCamelCase_					)
										_snake_case :			int  						=   self.shortcut(lowerCamelCase_					)
										hidden_state += residual
										_snake_case :			Union[str, Any]  						=   self.activation(lowerCamelCase_					)
										return hidden_state



class 			lowercase				(							tf.keras.layers.Layer     ):



						"""simple docstring"""

						def __init__(    self			:		List[Any] ,				lowerCamelCase_			:		RegNetConfig ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int ,				lowerCamelCase_			:		int = 2 ,				lowerCamelCase_			:		int = 2 ,				**lowerCamelCase_			:		Optional[Any]					):





										'''simple docstring'''







										super().__init__(**lowerCamelCase_					)

										_snake_case :			Tuple  						=   TFRegNetXLayer if config.layer_type == 'x' else TFRegNetYLayer
										_snake_case :			Tuple  						=   [
										    # downsampling is done in the first layer with stride of 2
										    layer(lowerCamelCase_ ,				lowerCamelCase_ ,				lowerCamelCase_ ,				stride=lowerCamelCase_ ,				name='layers.0'					),
										    *[layer(lowerCamelCase_ ,				lowerCamelCase_ ,				lowerCamelCase_ ,				name=f'''layers.{i+1}'''					) for i in range(depth - 1					)],
										]







						def 		__UpperCAmelCase       (    self			:		Tuple ,				lowerCamelCase_			:		Union[str, Any]					):





										'''simple docstring'''







										for layer_module in self.layers:
														_snake_case :			str  						=   layer_module(lowerCamelCase_					)
										return hidden_state



class 			lowercase				(							tf.keras.layers.Layer     ):



						"""simple docstring"""

						def __init__(    self			:		List[str] ,				lowerCamelCase_			:		RegNetConfig ,				**lowerCamelCase_			:		List[Any]					):





										'''simple docstring'''







										super().__init__(**lowerCamelCase_					)
										_snake_case :			int  						=   []
										# based on `downsample_in_first_stage`, the first layer of the first stage may or may not downsample the input
										self.stages.append(
										    TFRegNetStage(
										        lowerCamelCase_ ,				config.embedding_size ,				config.hidden_sizes[0] ,				stride=2 if config.downsample_in_first_stage else 1 ,				depth=config.depths[0] ,				name='stages.0' ,				)					)
										_snake_case :			Optional[int]  						=   zip(config.hidden_sizes ,				config.hidden_sizes[1:]					)
										for i, ((in_channels, out_channels), depth) in enumerate(zip(lowerCamelCase_ ,				config.depths[1:]					)					):
														self.stages.append(TFRegNetStage(lowerCamelCase_ ,				lowerCamelCase_ ,				lowerCamelCase_ ,				depth=lowerCamelCase_ ,				name=f'''stages.{i+1}'''					)					)







						def 		__UpperCAmelCase       (    self			:		List[str] ,				lowerCamelCase_			:		tf.Tensor ,				lowerCamelCase_			:		bool = False ,				lowerCamelCase_			:		bool = True					):





										'''simple docstring'''







										_snake_case :			Optional[Any]  						=   () if output_hidden_states else None

										for stage_module in self.stages:
														if output_hidden_states:
																		_snake_case :			Dict  						=   hidden_states + (hidden_state,)

														_snake_case :			Tuple  						=   stage_module(lowerCamelCase_					)

										if output_hidden_states:
														_snake_case :			str  						=   hidden_states + (hidden_state,)

										if not return_dict:
														return tuple(v for v in [hidden_state, hidden_states] if v is not None					)

										return TFBaseModelOutputWithNoAttention(last_hidden_state=lowerCamelCase_ ,				hidden_states=lowerCamelCase_					)



@keras_serializable
class 			lowercase				(							tf.keras.layers.Layer     ):



						"""simple docstring"""

						_UpperCamelCase  :	Union[str, Any]								=	RegNetConfig







						def __init__(    self			:		Dict ,				lowerCamelCase_			:		Tuple ,				**lowerCamelCase_			:		str					):





										'''simple docstring'''







										super().__init__(**lowerCamelCase_					)
										_snake_case :			Optional[int]  						=   config
										_snake_case :			Tuple  						=   TFRegNetEmbeddings(lowerCamelCase_ ,				name='embedder'					)
										_snake_case :			Optional[Any]  						=   TFRegNetEncoder(lowerCamelCase_ ,				name='encoder'					)
										_snake_case :			Union[str, Any]  						=   tf.keras.layers.GlobalAveragePoolingaD(keepdims=lowerCamelCase_ ,				name='pooler'					)







						@unpack_inputs
						def 		__UpperCAmelCase       (    self			:		Union[str, Any] ,				lowerCamelCase_			:		tf.Tensor ,				lowerCamelCase_			:		Optional[bool] = None ,				lowerCamelCase_			:		Optional[bool] = None ,				lowerCamelCase_			:		bool = False ,				):





										'''simple docstring'''







										_snake_case :			List[str]  						=   (
										    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
										)
										_snake_case :			Tuple  						=   return_dict if return_dict is not None else self.config.use_return_dict

										_snake_case :			List[str]  						=   self.embedder(lowerCamelCase_ ,				training=lowerCamelCase_					)

										_snake_case :			Union[str, Any]  						=   self.encoder(
										    lowerCamelCase_ ,				output_hidden_states=lowerCamelCase_ ,				return_dict=lowerCamelCase_ ,				training=lowerCamelCase_					)

										_snake_case :			Optional[int]  						=   encoder_outputs[0]
										_snake_case :			List[Any]  						=   self.pooler(lowerCamelCase_					)

										# Change to NCHW output format have uniformity in the modules
										_snake_case :			int  						=   tf.transpose(lowerCamelCase_ ,				perm=(0, 3, 1, 2)					)
										_snake_case :			Dict  						=   tf.transpose(lowerCamelCase_ ,				perm=(0, 3, 1, 2)					)

										# Change the other hidden state outputs to NCHW as well
										if output_hidden_states:
														_snake_case :			str  						=   tuple([tf.transpose(lowerCamelCase_ ,				perm=(0, 3, 1, 2)					) for h in encoder_outputs[1]]					)

										if not return_dict:
														return (last_hidden_state, pooled_output) + encoder_outputs[1:]

										return TFBaseModelOutputWithPoolingAndNoAttention(
										    last_hidden_state=lowerCamelCase_ ,				pooler_output=lowerCamelCase_ ,				hidden_states=hidden_states if output_hidden_states else encoder_outputs.hidden_states ,				)



class 			lowercase				(							a_     ):



						"""simple docstring"""

						_UpperCamelCase  :	str								=	RegNetConfig
						_UpperCamelCase  :	Tuple								=	"regnet"
						_UpperCamelCase  :	List[str]								=	"pixel_values"







						@property
						def 		__UpperCAmelCase       (    self			:		str					):





										'''simple docstring'''







										return {"pixel_values": tf.TensorSpec(shape=(None, self.config.num_channels, 2_24, 2_24) ,				dtype=tf.floataa					)}



lowercase_		:				int				=						r'''
    Parameters:
    This model is a Tensorflow
    [tf.keras.layers.Layer](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer) sub-class. Use it as a
    regular Tensorflow Module and refer to the Tensorflow documentation for all matter related to general usage and
    behavior.
        config ([`RegNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
'''

lowercase_		:				List[str]				=						r'''
    Args:
        pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`ConveNextImageProcessor.__call__`] for details.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
'''


@add_start_docstrings(
    "The bare RegNet model outputting raw features without any specific head on top."		,			a_		,			)
class 			lowercase				(							a_     ):



						"""simple docstring"""

						def __init__(    self			:		Tuple ,				lowerCamelCase_			:		RegNetConfig ,				*lowerCamelCase_			:		Union[str, Any] ,				**lowerCamelCase_			:		List[str]					):





										'''simple docstring'''







										super().__init__(lowerCamelCase_ ,				*lowerCamelCase_ ,				**lowerCamelCase_					)
										_snake_case :			Any  						=   TFRegNetMainLayer(lowerCamelCase_ ,				name='regnet'					)







						@unpack_inputs
						@add_start_docstrings_to_model_forward(lowerCamelCase_					)
						@add_code_sample_docstrings(
						    checkpoint=_CHECKPOINT_FOR_DOC ,				output_type=lowerCamelCase_ ,				config_class=_CONFIG_FOR_DOC ,				modality='vision' ,				expected_output=_EXPECTED_OUTPUT_SHAPE ,				)
						def 		__UpperCAmelCase       (    self			:		int ,				lowerCamelCase_			:		tf.Tensor ,				lowerCamelCase_			:		Optional[bool] = None ,				lowerCamelCase_			:		Optional[bool] = None ,				lowerCamelCase_			:		Optional[int]=False ,				):





										'''simple docstring'''







										_snake_case :			List[Any]  						=   (
										    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
										)
										_snake_case :			int  						=   return_dict if return_dict is not None else self.config.use_return_dict

										_snake_case :			List[str]  						=   self.regnet(
										    pixel_values=lowerCamelCase_ ,				output_hidden_states=lowerCamelCase_ ,				return_dict=lowerCamelCase_ ,				training=lowerCamelCase_ ,				)
										if not return_dict:
														return (outputs[0],) + outputs[1:]

										return TFBaseModelOutputWithPoolingAndNoAttention(
										    last_hidden_state=outputs.last_hidden_state ,				pooler_output=outputs.pooler_output ,				hidden_states=outputs.hidden_states ,				)



@add_start_docstrings(
    "\n    RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    "		,			a_		,			)
class 			lowercase				(							a_		,			a_     ):



						"""simple docstring"""

						def __init__(    self			:		Dict ,				lowerCamelCase_			:		RegNetConfig ,				*lowerCamelCase_			:		Dict ,				**lowerCamelCase_			:		int					):





										'''simple docstring'''







										super().__init__(lowerCamelCase_ ,				*lowerCamelCase_ ,				**lowerCamelCase_					)
										_snake_case :			Any  						=   config.num_labels
										_snake_case :			Optional[int]  						=   TFRegNetMainLayer(lowerCamelCase_ ,				name='regnet'					)
										# classification head
										_snake_case :			Any  						=   [
										    tf.keras.layers.Flatten(),
										    tf.keras.layers.Dense(config.num_labels ,				name='classifier.1'					) if config.num_labels > 0 else tf.identity,
										]







						@unpack_inputs
						@add_start_docstrings_to_model_forward(lowerCamelCase_					)
						@add_code_sample_docstrings(
						    checkpoint=_IMAGE_CLASS_CHECKPOINT ,				output_type=lowerCamelCase_ ,				config_class=_CONFIG_FOR_DOC ,				expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT ,				)
						def 		__UpperCAmelCase       (    self			:		Dict ,				lowerCamelCase_			:		tf.Tensor = None ,				lowerCamelCase_			:		tf.Tensor = None ,				lowerCamelCase_			:		bool = None ,				lowerCamelCase_			:		bool = None ,				lowerCamelCase_			:		Tuple=False ,				):





										'''simple docstring'''







										_snake_case :			List[Any]  						=   (
										    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
										)
										_snake_case :			str  						=   return_dict if return_dict is not None else self.config.use_return_dict

										_snake_case :			Optional[Any]  						=   self.regnet(
										    lowerCamelCase_ ,				output_hidden_states=lowerCamelCase_ ,				return_dict=lowerCamelCase_ ,				training=lowerCamelCase_					)

										_snake_case :			Union[str, Any]  						=   outputs.pooler_output if return_dict else outputs[1]

										_snake_case :			Tuple  						=   self.classifier[0](lowerCamelCase_					)
										_snake_case :			Tuple  						=   self.classifier[1](lowerCamelCase_					)

										_snake_case :			Dict  						=   None if labels is None else self.hf_compute_loss(labels=lowerCamelCase_ ,				logits=lowerCamelCase_					)

										if not return_dict:
														_snake_case :			str  						=   (logits,) + outputs[2:]
														return ((loss,) + output) if loss is not None else output

										return TFSequenceClassifierOutput(loss=lowerCamelCase_ ,				logits=lowerCamelCase_ ,				hidden_states=outputs.hidden_states					)













import torch

from transformers import AutoModel


class 			lowercase				(							torch.nn.Module     ):



						"""simple docstring"""

						def __init__(    self			:		Tuple ,				lowerCamelCase_			:		Dict="sayef/fsner-bert-base-uncased"					):





										'''simple docstring'''







										super(lowerCamelCase_ ,				self					).__init__()

										_snake_case :			Optional[Any]  						=   AutoModel.from_pretrained(lowerCamelCase_ ,				return_dict=lowerCamelCase_					)
										_snake_case :			str  						=   torch.nn.CosineSimilarity(3 ,				1e-08					)
										_snake_case :			str  						=   torch.nn.Softmax(dim=1					)







						def 		__UpperCAmelCase       (    self			:		int ,				**lowerCamelCase_			:		List[str]					):





										'''simple docstring'''







										return self.bert(**lowerCamelCase_					).last_hidden_state







						def 		__UpperCAmelCase       (    self			:		List[Any] ,				lowerCamelCase_			:		Union[str, Any]					):





										'''simple docstring'''







										return token_embeddings.sum(2 ,				keepdim=lowerCamelCase_					)







						def 		__UpperCAmelCase       (    self			:		Dict ,				lowerCamelCase_			:		Dict ,				lowerCamelCase_			:		Tuple ,				lowerCamelCase_			:		Optional[Any]=1					):





										'''simple docstring'''







										return self.softmax(T * self.cos(lowerCamelCase_ ,				lowerCamelCase_					)					)







						def 		__UpperCAmelCase       (    self			:		Optional[Any] ,				lowerCamelCase_			:		Any ,				lowerCamelCase_			:		Union[str, Any]					):





										'''simple docstring'''







										_snake_case :			int  						=   W_supports['sizes'].tolist()
										_snake_case :			int  						=   W_supports['start_token_id'].item()
										_snake_case :			List[str]  						=   W_supports['end_token_id'].item()

										del W_supports["sizes"]
										del W_supports["start_token_id"]
										del W_supports["end_token_id"]

										_snake_case :			Optional[int]  						=   self.BERT(**lowerCamelCase_					)
										_snake_case :			Optional[Any]  						=   self.BERT(**lowerCamelCase_					)

										_snake_case :			Optional[int]  						=   None
										_snake_case :			Optional[int]  						=   None

										_snake_case :			List[str]  						=   W_supports['input_ids'] == start_token_id
										_snake_case :			Union[str, Any]  						=   W_supports['input_ids'] == end_token_id

										for i, size in enumerate(lowerCamelCase_					):
														if i == 0:
																		_snake_case :			str  						=   0
														else:
																		_snake_case :			Union[str, Any]  						=   support_sizes[i - 1]

														_snake_case :			Tuple  						=   S[s : s + size][start_token_masks[s : s + size]]
														_snake_case :			Optional[int]  						=   S[s : s + size][end_token_masks[s : s + size]]

														_snake_case :			Tuple  						=   torch.matmul(q[i] ,				s_start.T					).sum(1					).softmax(0					)
														_snake_case :			Union[str, Any]  						=   torch.matmul(q[i] ,				s_end.T					).sum(1					).softmax(0					)

														if p_starts is not None:
																		_snake_case :			Optional[Any]  						=   torch.vstack((p_starts, p_start)					)
																		_snake_case :			List[str]  						=   torch.vstack((p_ends, p_end)					)
														else:
																		_snake_case :			Union[str, Any]  						=   p_start
																		_snake_case :			Any  						=   p_end

										return p_starts, p_ends






