




'''simple docstring'''





from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy




@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """					, _UpperCamelCase					, )
class 					snake_case__							(_UpperCamelCase     ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_ :							Union[str, Any]											=      RobertaConfig
				SCREAMING_SNAKE_CASE_ :							Optional[Any]											=      """roberta"""





				def __init__(   self       :  List[str]      ,   __lowerCamelCase       :  Union[str, Any]							)      ->  Tuple:
							super().__init__(__lowerCamelCase							)

							a					=    RobertaEmbeddings(__lowerCamelCase							)
							self.init_weights()




@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """					, _UpperCamelCase					, )
class 					snake_case__							(_UpperCamelCase     ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_ :							Union[str, Any]											=      RobertaConfig
				SCREAMING_SNAKE_CASE_ :							int											=      """roberta"""





				def __init__(   self       :  List[str]      ,   __lowerCamelCase       :  Tuple							)      ->  Optional[int]:
							super().__init__(__lowerCamelCase							)
							a					=    config.num_labels
							a					=    config.num_hidden_layers

							a					=    DeeRobertaModel(__lowerCamelCase							)
							a					=    nn.Dropout(config.hidden_dropout_prob							)
							a					=    nn.Linear(config.hidden_size      ,   self.config.num_labels							)





				@add_start_docstrings_to_model_forward(__lowerCamelCase							)
				def 							__UpperCAmelCase							(   self       :  Any      ,   __lowerCamelCase       :  Union[str, Any]=None      ,   __lowerCamelCase       :  List[str]=None      ,   __lowerCamelCase       :  Optional[Any]=None      ,   __lowerCamelCase       :  Optional[Any]=None      ,   __lowerCamelCase       :  int=None      ,   __lowerCamelCase       :  Any=None      ,   __lowerCamelCase       :  Union[str, Any]=None      ,   __lowerCamelCase       :  int=-1      ,   __lowerCamelCase       :  Tuple=False      ,   )      ->  str:
							a					=    self.num_layers
							try:
										a					=    self.roberta(
										    __lowerCamelCase      ,   attention_mask=__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   position_ids=__lowerCamelCase      ,   head_mask=__lowerCamelCase      ,   inputs_embeds=__lowerCamelCase      ,   )

										a					=    outputs[1]

										a					=    self.dropout(__lowerCamelCase							)
										a					=    self.classifier(__lowerCamelCase							)
										a					=    (logits,) + outputs[2:]  # add hidden states and attention if they are here
							except HighwayException as e:
										a					=    e.message
										a					=    e.exit_layer
										a					=    outputs[0]

							if not self.training:
										a					=    entropy(__lowerCamelCase							)
										a					=    []
										a					=    []
							if labels is not None:
										if self.num_labels == 1:
													#  We are doing regression
													a					=    MSELoss()
													a					=    loss_fct(logits.view(-1							)      ,   labels.view(-1							)							)
										else:
													a					=    CrossEntropyLoss()
													a					=    loss_fct(logits.view(-1      ,   self.num_labels							)      ,   labels.view(-1							)							)

										# work with highway exits
										a					=    []
										for highway_exit in outputs[-1]:
													a					=    highway_exit[0]
													if not self.training:
																highway_logits_all.append(__lowerCamelCase							)
																highway_entropy.append(highway_exit[2]							)
													if self.num_labels == 1:
																#  We are doing regression
																a					=    MSELoss()
																a					=    loss_fct(highway_logits.view(-1							)      ,   labels.view(-1							)							)
													else:
																a					=    CrossEntropyLoss()
																a					=    loss_fct(highway_logits.view(-1      ,   self.num_labels							)      ,   labels.view(-1							)							)
													highway_losses.append(__lowerCamelCase							)

										if train_highway:
													a					=    (sum(highway_losses[:-1]							),) + outputs
													# exclude the final highway, of course
										else:
													a					=    (loss,) + outputs
							if not self.training:
										a					=    outputs + ((original_entropy, highway_entropy), exit_layer)
										if output_layer >= 0:
													a					=    (
													    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
													)  # use the highway of the last layer

							return outputs  # (loss), logits, (hidden_states), (attentions), entropy












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__lowerCAmelCase	:       Any     =	{
    'configuration_roc_bert': ['ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'RoCBertConfig'],
    'tokenization_roc_bert': ['RoCBertTokenizer'],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		pass

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__lowerCAmelCase	:       Optional[Any]     =	[
		    'ROC_BERT_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'RoCBertForCausalLM',
		    'RoCBertForMaskedLM',
		    'RoCBertForMultipleChoice',
		    'RoCBertForPreTraining',
		    'RoCBertForQuestionAnswering',
		    'RoCBertForSequenceClassification',
		    'RoCBertForTokenClassification',
		    'RoCBertLayer',
		    'RoCBertModel',
		    'RoCBertPreTrainedModel',
		    'load_tf_weights_in_roc_bert',
		]

if TYPE_CHECKING:
		from .configuration_roc_bert import ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RoCBertConfig
		from .tokenization_roc_bert import RoCBertTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				raise OptionalDependencyNotAvailable()

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_roc_bert import (
				    ROC_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    RoCBertForCausalLM,
				    RoCBertForMaskedLM,
				    RoCBertForMultipleChoice,
				    RoCBertForPreTraining,
				    RoCBertForQuestionAnswering,
				    RoCBertForSequenceClassification,
				    RoCBertForTokenClassification,
				    RoCBertLayer,
				    RoCBertModel,
				    RoCBertPreTrainedModel,
				    load_tf_weights_in_roc_bert,
				)


else:
		import sys

		__lowerCAmelCase	:       Union[str, Any]     =	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






