"""simple docstring"""






from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy
@add_start_docstrings(
    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). "   ,					__UpperCAmelCase   ,					)
class        snake_case (				__UpperCAmelCase       ):




					"""simple docstring"""

					snake_case__       					=				RobertaConfig
					snake_case__       					=				"roberta"







					def __init__(       self     :			List[str]					,lowerCamelCase__     :			Dict    ):
									super().__init__(lowerCamelCase__    )

									UpperCAmelCase__    =							RobertaEmbeddings(lowerCamelCase__    )
									self.init_weights()






@add_start_docstrings(
    "RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. "   ,					__UpperCAmelCase   ,					)
class        snake_case (				__UpperCAmelCase       ):




					"""simple docstring"""

					snake_case__       					=				RobertaConfig
					snake_case__       					=				"roberta"







					def __init__(       self     :			Union[str, Any]					,lowerCamelCase__     :			Optional[int]    ):
									super().__init__(lowerCamelCase__    )
									UpperCAmelCase__    =							config.num_labels
									UpperCAmelCase__    =							config.num_hidden_layers

									UpperCAmelCase__    =							DeeRobertaModel(lowerCamelCase__    )
									UpperCAmelCase__    =							nn.Dropout(config.hidden_dropout_prob    )
									UpperCAmelCase__    =							nn.Linear(config.hidden_size					,self.config.num_labels    )







					@add_start_docstrings_to_model_forward(lowerCamelCase__    )
					def 	__lowerCAmelCase		(       self     :			str					,lowerCamelCase__     :			Union[str, Any]=None					,lowerCamelCase__     :			List[str]=None					,lowerCamelCase__     :			str=None					,lowerCamelCase__     :			Tuple=None					,lowerCamelCase__     :			List[Any]=None					,lowerCamelCase__     :			Optional[Any]=None					,lowerCamelCase__     :			Tuple=None					,lowerCamelCase__     :			int=-1					,lowerCamelCase__     :			Any=False					,):
									UpperCAmelCase__    =							self.num_layers
									try:
													UpperCAmelCase__    =							self.roberta(
													    lowerCamelCase__					,attention_mask=lowerCamelCase__					,token_type_ids=lowerCamelCase__					,position_ids=lowerCamelCase__					,head_mask=lowerCamelCase__					,inputs_embeds=lowerCamelCase__					,)

													UpperCAmelCase__    =							outputs[1]

													UpperCAmelCase__    =							self.dropout(lowerCamelCase__    )
													UpperCAmelCase__    =							self.classifier(lowerCamelCase__    )
													UpperCAmelCase__    =							(logits,) + outputs[2:]  # add hidden states and attention if they are here
									except HighwayException as e:
													UpperCAmelCase__    =							e.message
													UpperCAmelCase__    =							e.exit_layer
													UpperCAmelCase__    =							outputs[0]

									if not self.training:
													UpperCAmelCase__    =							entropy(lowerCamelCase__    )
													UpperCAmelCase__    =							[]
													UpperCAmelCase__    =							[]
									if labels is not None:
													if self.num_labels == 1:
																	#  We are doing regression
																	UpperCAmelCase__    =							MSELoss()
																	UpperCAmelCase__    =							loss_fct(logits.view(-1    )					,labels.view(-1    )    )
													else:
																	UpperCAmelCase__    =							CrossEntropyLoss()
																	UpperCAmelCase__    =							loss_fct(logits.view(-1					,self.num_labels    )					,labels.view(-1    )    )

													# work with highway exits
													UpperCAmelCase__    =							[]
													for highway_exit in outputs[-1]:
																	UpperCAmelCase__    =							highway_exit[0]
																	if not self.training:
																					highway_logits_all.append(lowerCamelCase__    )
																					highway_entropy.append(highway_exit[2]    )
																	if self.num_labels == 1:
																					#  We are doing regression
																					UpperCAmelCase__    =							MSELoss()
																					UpperCAmelCase__    =							loss_fct(highway_logits.view(-1    )					,labels.view(-1    )    )
																	else:
																					UpperCAmelCase__    =							CrossEntropyLoss()
																					UpperCAmelCase__    =							loss_fct(highway_logits.view(-1					,self.num_labels    )					,labels.view(-1    )    )
																	highway_losses.append(lowerCamelCase__    )

													if train_highway:
																	UpperCAmelCase__    =							(sum(highway_losses[:-1]    ),) + outputs
																	# exclude the final highway, of course
													else:
																	UpperCAmelCase__    =							(loss,) + outputs
									if not self.training:
													UpperCAmelCase__    =							outputs + ((original_entropy, highway_entropy), exit_layer)
													if output_layer >= 0:
																	UpperCAmelCase__    =							(
																	    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																	)  # use the highway of the last layer

									return outputs  # (loss), logits, (hidden_states), (attentions), entropy


"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__						:						Any		  =			logging.get_logger(__name__)

lowerCAmelCase__						:						str		  =			{'ctrl': 'https://huggingface.co/ctrl/resolve/main/config.json'}
class        snake_case (				__UpperCAmelCase       ):




					"""simple docstring"""

					snake_case__       					=				"ctrl"
					snake_case__       					=				["past_key_values"]
					snake_case__       					=				{
					    "max_position_embeddings": "n_positions",
					    "hidden_size": "n_embd",
					    "num_attention_heads": "n_head",
					    "num_hidden_layers": "n_layer",
					}







					def __init__(       self     :			Any					,lowerCamelCase__     :			str=246_534					,lowerCamelCase__     :			List[str]=256					,lowerCamelCase__     :			Optional[int]=1_280					,lowerCamelCase__     :			Any=8_192					,lowerCamelCase__     :			int=48					,lowerCamelCase__     :			Optional[Any]=16					,lowerCamelCase__     :			Union[str, Any]=0.1					,lowerCamelCase__     :			Dict=0.1					,lowerCamelCase__     :			List[str]=1e-6					,lowerCamelCase__     :			List[str]=0.0_2					,lowerCamelCase__     :			Tuple=True					,**lowerCamelCase__     :			Optional[Any]					,):
									UpperCAmelCase__    =							vocab_size
									UpperCAmelCase__    =							n_positions
									UpperCAmelCase__    =							n_embd
									UpperCAmelCase__    =							n_layer
									UpperCAmelCase__    =							n_head
									UpperCAmelCase__    =							dff
									UpperCAmelCase__    =							resid_pdrop
									UpperCAmelCase__    =							embd_pdrop
									UpperCAmelCase__    =							layer_norm_epsilon
									UpperCAmelCase__    =							initializer_range

									UpperCAmelCase__    =							use_cache

									super().__init__(**lowerCamelCase__    )



