





'''simple docstring'''




from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy






@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """					,			__snake_case					,			)
class UpperCAmelCase							(		__snake_case							):
    _A :			Union[str, Any]						   =       RobertaConfig
    _A :			Optional[int]						   =       'roberta'



    def __init__(  self     ,			__A					):
     super().__init__(A_					)

     __UpperCAmelCase					=   RobertaEmbeddings(A_					)
     self.init_weights()






@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. """					,			__snake_case					,			)
class UpperCAmelCase							(		__snake_case							):
    _A :			Optional[Any]						   =       RobertaConfig
    _A :			List[str]						   =       'roberta'



    def __init__(  self     ,			__A					):
     super().__init__(A_					)
     __UpperCAmelCase					=   config.num_labels
     __UpperCAmelCase					=   config.num_hidden_layers

     __UpperCAmelCase					=   DeeRobertaModel(A_					)
     __UpperCAmelCase					=   nn.Dropout(config.hidden_dropout_prob					)
     __UpperCAmelCase					=   nn.Linear(config.hidden_size     ,			self.config.num_labels					)




    @add_start_docstrings_to_model_forward(A_					)
    def 					__lowerCamelCase						(  self     ,			__A=None     ,			__A=None     ,			__A=None     ,			__A=None     ,			__A=None     ,			__A=None     ,			__A=None     ,			__A=-1     ,			__A=False     ,			):
     __UpperCAmelCase					=   self.num_layers
     try:
      __UpperCAmelCase					=   self.roberta(
          A_     ,			attention_mask=A_     ,			token_type_ids=A_     ,			position_ids=A_     ,			head_mask=A_     ,			inputs_embeds=A_     ,			)

      __UpperCAmelCase					=   outputs[1]

      __UpperCAmelCase					=   self.dropout(A_					)
      __UpperCAmelCase					=   self.classifier(A_					)
      __UpperCAmelCase					=   (logits,) + outputs[2:]  # add hidden states and attention if they are here
     except HighwayException as e:
      __UpperCAmelCase					=   e.message
      __UpperCAmelCase					=   e.exit_layer
      __UpperCAmelCase					=   outputs[0]

     if not self.training:
      __UpperCAmelCase					=   entropy(A_					)
      __UpperCAmelCase					=   []
      __UpperCAmelCase					=   []
     if labels is not None:
      if self.num_labels == 1:
       #  We are doing regression
       __UpperCAmelCase					=   MSELoss()
       __UpperCAmelCase					=   loss_fct(logits.view(-1					)     ,			labels.view(-1					)					)
      else:
       __UpperCAmelCase					=   CrossEntropyLoss()
       __UpperCAmelCase					=   loss_fct(logits.view(-1     ,			self.num_labels					)     ,			labels.view(-1					)					)

      # work with highway exits
      __UpperCAmelCase					=   []
      for highway_exit in outputs[-1]:
       __UpperCAmelCase					=   highway_exit[0]
       if not self.training:
        highway_logits_all.append(A_					)
        highway_entropy.append(highway_exit[2]					)
       if self.num_labels == 1:
        #  We are doing regression
        __UpperCAmelCase					=   MSELoss()
        __UpperCAmelCase					=   loss_fct(highway_logits.view(-1					)     ,			labels.view(-1					)					)
       else:
        __UpperCAmelCase					=   CrossEntropyLoss()
        __UpperCAmelCase					=   loss_fct(highway_logits.view(-1     ,			self.num_labels					)     ,			labels.view(-1					)					)
       highway_losses.append(A_					)

      if train_highway:
       __UpperCAmelCase					=   (sum(highway_losses[:-1]					),) + outputs
       # exclude the final highway, of course
      else:
       __UpperCAmelCase					=   (loss,) + outputs
     if not self.training:
      __UpperCAmelCase					=   outputs + ((original_entropy, highway_entropy), exit_layer)
      if output_layer >= 0:
       __UpperCAmelCase					=   (
           (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
       )  # use the highway of the last layer

     return outputs  # (loss), logits, (hidden_states), (attentions), entropy












'''simple docstring'''




from typing import List, Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A:					str							=				logging.get_logger(__name__)

_A:					Optional[Any]							=				{
    """huggingface/autoformer-tourism-monthly""": """https://huggingface.co/huggingface/autoformer-tourism-monthly/resolve/main/config.json""",
}






class UpperCAmelCase							(		UpperCAmelCase_							):
    _A :			Tuple						   =       """autoformer"""
    _A :			str						   =       {
        """hidden_size""": """d_model""",
        """num_attention_heads""": """encoder_attention_heads""",
        """num_hidden_layers""": """encoder_layers""",
    }



    def __init__(  self     ,			__A = None     ,			__A = None     ,			__A = "student_t"     ,			__A = "nll"     ,			__A = 1     ,			__A = [1, 2, 3, 4, 5, 6, 7]     ,			__A = True     ,			__A = 0     ,			__A = 0     ,			__A = 0     ,			__A = 0     ,			__A = None     ,			__A = None     ,			__A = 64     ,			__A = 2     ,			__A = 2     ,			__A = 2     ,			__A = 2     ,			__A = 32     ,			__A = 32     ,			__A = "gelu"     ,			__A = 0.1     ,			__A = 0.1     ,			__A = 0.1     ,			__A = 0.1     ,			__A = 0.1     ,			__A = 100     ,			__A = 0.0_2     ,			__A = True     ,			__A=True     ,			__A = 10     ,			__A = 25     ,			__A = 3     ,			**__A     ,			):
     # time series specific configuration
     __UpperCAmelCase					=   prediction_length
     __UpperCAmelCase					=   context_length if context_length is not None else prediction_length
     __UpperCAmelCase					=   distribution_output
     __UpperCAmelCase					=   loss
     __UpperCAmelCase					=   input_size
     __UpperCAmelCase					=   num_time_features
     __UpperCAmelCase					=   lags_sequence
     __UpperCAmelCase					=   scaling
     __UpperCAmelCase					=   num_dynamic_real_features
     __UpperCAmelCase					=   num_static_real_features
     __UpperCAmelCase					=   num_static_categorical_features
     if cardinality is not None and num_static_categorical_features > 0:
      if len(__A					) != num_static_categorical_features:
       raise ValueError(
           'The cardinality should be a list of the same length as `num_static_categorical_features`'					)
      __UpperCAmelCase					=   cardinality
     else:
      __UpperCAmelCase					=   [0]
     if embedding_dimension is not None and num_static_categorical_features > 0:
      if len(__A					) != num_static_categorical_features:
       raise ValueError(
           'The embedding dimension should be a list of the same length as `num_static_categorical_features`'					)
      __UpperCAmelCase					=   embedding_dimension
     else:
      __UpperCAmelCase					=   [min(50     ,			(cat + 1) // 2					) for cat in self.cardinality]
     __UpperCAmelCase					=   num_parallel_samples

     # Transformer architecture configuration
     __UpperCAmelCase					=   input_size * len(self.lags_sequence					) + self._number_of_features
     __UpperCAmelCase					=   d_model
     __UpperCAmelCase					=   encoder_attention_heads
     __UpperCAmelCase					=   decoder_attention_heads
     __UpperCAmelCase					=   encoder_ffn_dim
     __UpperCAmelCase					=   decoder_ffn_dim
     __UpperCAmelCase					=   encoder_layers
     __UpperCAmelCase					=   decoder_layers

     __UpperCAmelCase					=   dropout
     __UpperCAmelCase					=   attention_dropout
     __UpperCAmelCase					=   activation_dropout
     __UpperCAmelCase					=   encoder_layerdrop
     __UpperCAmelCase					=   decoder_layerdrop

     __UpperCAmelCase					=   activation_function
     __UpperCAmelCase					=   init_std

     __UpperCAmelCase					=   use_cache

     # Autoformer
     __UpperCAmelCase					=   label_length
     __UpperCAmelCase					=   moving_average
     __UpperCAmelCase					=   autocorrelation_factor

     super().__init__(is_encoder_decoder=__A     ,			**__A					)




    @property
    def 					__lowerCamelCase						(  self					):
     return (
         sum(self.embedding_dimension					)
         + self.num_dynamic_real_features
         + self.num_time_features
         + self.num_static_real_features
         + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
     )





