



import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


_snake_case	=						logging.getLogger(__name__)



class 						UpperCAmelCase_					( __UpperCAmelCase):







       def      snake_case__   (     self,    __a,    __a,    __a=None,    __a=None):

              '''simple docstring'''

              _lowerCAmelCase  :							Dict				     =					self.layer[current_layer](__a,    __a,    head_mask[current_layer])

              _lowerCAmelCase  :							Union[str, Any]				     =					layer_outputs[0]

              return hidden_states



@add_start_docstrings(
    'The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'     ,      __UpperCAmelCase     ,      )
class 						UpperCAmelCase_					( __UpperCAmelCase):







       def __init__(     self,    __a):

              '''simple docstring'''

              super().__init__(__a)

              _lowerCAmelCase  :							Tuple				     =					BertEncoderWithPabee(__a)

              self.init_weights()
              _lowerCAmelCase  :							Tuple				     =					0
              _lowerCAmelCase  :							str				     =					0
              _lowerCAmelCase  :							List[str]				     =					0

              _lowerCAmelCase  :							str				     =					0







       def      snake_case__   (     self,    __a):

              '''simple docstring'''

              _lowerCAmelCase  :							Dict				     =					threshold







       def      snake_case__   (     self,    __a):

              '''simple docstring'''

              _lowerCAmelCase  :							Optional[int]				     =					patience







       def      snake_case__   (     self):

              '''simple docstring'''

              _lowerCAmelCase  :							Union[str, Any]				     =					0
              _lowerCAmelCase  :							Optional[Any]				     =					0







       def      snake_case__   (     self):

              '''simple docstring'''

              _lowerCAmelCase  :							Any				     =					self.inference_layers_num / self.inference_instances_num
              _lowerCAmelCase  :							List[Any]				     =					(
                  f"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
                  f" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
              )
              print(__a)







       @add_start_docstrings_to_model_forward(__a)
       def      snake_case__   (     self,    __a=None,    __a=None,    __a=None,    __a=None,    __a=None,    __a=None,    __a=None,    __a=None,    __a=None,    __a=None,    __a=False,    ):

              '''simple docstring'''

              if input_ids is not None and inputs_embeds is not None:
                     raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
              elif input_ids is not None:
                     _lowerCAmelCase  :							Tuple				     =					input_ids.size()
              elif inputs_embeds is not None:
                     _lowerCAmelCase  :							Tuple				     =					inputs_embeds.size()[:-1]
              else:
                     raise ValueError("You have to specify either input_ids or inputs_embeds")

              _lowerCAmelCase  :							Optional[int]				     =					input_ids.device if input_ids is not None else inputs_embeds.device

              if attention_mask is None:
                     _lowerCAmelCase  :							str				     =					torch.ones(__a,    device=__a)
              if token_type_ids is None:
                     _lowerCAmelCase  :							Optional[int]				     =					torch.zeros(__a,    dtype=torch.long,    device=__a)

              # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
              # ourselves in which case we just need to make it broadcastable to all heads.
              _lowerCAmelCase  :							Optional[int]				     =					self.get_extended_attention_mask(__a,    __a,    __a)

              # If a 2D ou 3D attention mask is provided for the cross-attention
              # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
              if self.config.is_decoder and encoder_hidden_states is not None:
                     _lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase  :							Dict				     =					encoder_hidden_states.size()
                     _lowerCAmelCase  :							Optional[Any]				     =					(encoder_batch_size, encoder_sequence_length)
                     if encoder_attention_mask is None:
                            _lowerCAmelCase  :							Any				     =					torch.ones(__a,    device=__a)
                     _lowerCAmelCase  :							Union[str, Any]				     =					self.invert_attention_mask(__a)
              else:
                     _lowerCAmelCase  :							Optional[int]				     =					None

              # Prepare head mask if needed
              # 1.0 in head_mask indicate we keep the head
              # attention_probs has shape bsz x n_heads x N x N
              # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
              # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
              _lowerCAmelCase  :							Optional[int]				     =					self.get_head_mask(__a,    self.config.num_hidden_layers)

              _lowerCAmelCase  :							str				     =					self.embeddings(
                  input_ids=__a,    position_ids=__a,    token_type_ids=__a,    inputs_embeds=__a)
              _lowerCAmelCase  :							Union[str, Any]				     =					embedding_output

              if self.training:
                     _lowerCAmelCase  :							Dict				     =					[]
                     for i in range(self.config.num_hidden_layers):
                            _lowerCAmelCase  :							str				     =					self.encoder.adaptive_forward(
                                __a,    current_layer=__a,    attention_mask=__a,    head_mask=__a)

                            _lowerCAmelCase  :							Dict				     =					self.pooler(__a)
                            _lowerCAmelCase  :							Tuple				     =					output_layers[i](output_dropout(__a))
                            res.append(__a)
              elif self.patience == 0:  # Use all layers for inference
                     _lowerCAmelCase  :							List[str]				     =					self.encoder(
                         __a,    attention_mask=__a,    head_mask=__a,    encoder_hidden_states=__a,    encoder_attention_mask=__a,    )
                     _lowerCAmelCase  :							Union[str, Any]				     =					self.pooler(encoder_outputs[0])
                     _lowerCAmelCase  :							int				     =					[output_layers[self.config.num_hidden_layers - 1](__a)]
              else:
                     _lowerCAmelCase  :							List[str]				     =					0
                     _lowerCAmelCase  :							Tuple				     =					None
                     _lowerCAmelCase  :							List[Any]				     =					0
                     for i in range(self.config.num_hidden_layers):
                            calculated_layer_num += 1
                            _lowerCAmelCase  :							List[Any]				     =					self.encoder.adaptive_forward(
                                __a,    current_layer=__a,    attention_mask=__a,    head_mask=__a)

                            _lowerCAmelCase  :							int				     =					self.pooler(__a)
                            _lowerCAmelCase  :							Optional[Any]				     =					output_layers[i](__a)
                            if regression:
                                   _lowerCAmelCase  :							List[Any]				     =					logits.detach()
                                   if patient_result is not None:
                                          _lowerCAmelCase  :							str				     =					patient_result.detach()
                                   if (patient_result is not None) and torch.abs(patient_result - labels) < self.regression_threshold:
                                          patient_counter += 1
                                   else:
                                          _lowerCAmelCase  :							Union[str, Any]				     =					0
                            else:
                                   _lowerCAmelCase  :							Any				     =					logits.detach().argmax(dim=1)
                                   if patient_result is not None:
                                          _lowerCAmelCase  :							Tuple				     =					patient_result.detach().argmax(dim=1)
                                   if (patient_result is not None) and torch.all(labels.eq(__a)):
                                          patient_counter += 1
                                   else:
                                          _lowerCAmelCase  :							Optional[Any]				     =					0

                            _lowerCAmelCase  :							Any				     =					logits
                            if patient_counter == self.patience:
                                   break
                     _lowerCAmelCase  :							Tuple				     =					[patient_result]
                     self.inference_layers_num += calculated_layer_num
                     self.inference_instances_num += 1

              return res



@add_start_docstrings(
    'Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '     ,      __UpperCAmelCase     ,      )
class 						UpperCAmelCase_					( __UpperCAmelCase):







       def __init__(     self,    __a):

              '''simple docstring'''

              super().__init__(__a)
              _lowerCAmelCase  :							Any				     =					config.num_labels

              _lowerCAmelCase  :							Optional[Any]				     =					BertModelWithPabee(__a)
              _lowerCAmelCase  :							int				     =					nn.Dropout(config.hidden_dropout_prob)
              _lowerCAmelCase  :							List[Any]				     =					nn.ModuleList(
                  [nn.Linear(config.hidden_size,    self.config.num_labels) for _ in range(config.num_hidden_layers)])

              self.init_weights()







       @add_start_docstrings_to_model_forward(__a)
       def      snake_case__   (     self,    __a=None,    __a=None,    __a=None,    __a=None,    __a=None,    __a=None,    __a=None,    ):

              '''simple docstring'''

              _lowerCAmelCase  :							Tuple				     =					self.bert(
                  input_ids=__a,    attention_mask=__a,    token_type_ids=__a,    position_ids=__a,    head_mask=__a,    inputs_embeds=__a,    output_dropout=self.dropout,    output_layers=self.classifiers,    regression=self.num_labels == 1,    )

              _lowerCAmelCase  :							List[str]				     =					(logits[-1],)

              if labels is not None:
                     _lowerCAmelCase  :							Optional[Any]				     =					None
                     _lowerCAmelCase  :							List[str]				     =					0
                     for ix, logits_item in enumerate(__a):
                            if self.num_labels == 1:
                                   #  We are doing regression
                                   _lowerCAmelCase  :							str				     =					MSELoss()
                                   _lowerCAmelCase  :							Tuple				     =					loss_fct(logits_item.view(-1),    labels.view(-1))
                            else:
                                   _lowerCAmelCase  :							int				     =					CrossEntropyLoss()
                                   _lowerCAmelCase  :							List[Any]				     =					loss_fct(logits_item.view(-1,    self.num_labels),    labels.view(-1))
                            if total_loss is None:
                                   _lowerCAmelCase  :							List[Any]				     =					loss
                            else:
                                   total_loss += loss * (ix + 1)
                            total_weights += ix + 1
                     _lowerCAmelCase  :							Tuple				     =					(total_loss / total_weights,) + outputs

              return outputs





import baseaa






def        A       (      _lowerCamelCase						):





							'''simple docstring'''




							return baseaa.aaaencode(string.encode("utf-8"						)						)






def        A       (      _lowerCamelCase						):





							'''simple docstring'''




							return baseaa.aaadecode(_lowerCamelCase						).decode("utf-8"						)


if __name__ == "__main__":
						import doctest

						doctest.testmod()
