import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


_lowerCamelCase					:   Any							=		logging.getLogger(__name__)


class 						lowercase    (     a     ):




							def 	__snake_case(						self   :					Any	,							_UpperCamelCase   :					str	,							_UpperCamelCase   :					str	,							_UpperCamelCase   :					Optional[Any]=None	,							_UpperCamelCase   :					Any=None      )					->				Any:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       self.layer[current_layer](_UpperCamelCase	,							_UpperCamelCase	,							head_mask[current_layer]      )

												SCREAMING_SNAKE_CASE    			=       layer_outputs[0]

												return hidden_states



@add_start_docstrings(
    """The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."""				,       a				,       )
class 						lowercase    (     a     ):




							def __init__(						self   :					Dict	,							_UpperCamelCase   :					str      )					->				int:



												'''simple docstring'''

												super().__init__(_UpperCamelCase      )

												SCREAMING_SNAKE_CASE    			=       BertEncoderWithPabee(_UpperCamelCase      )

												self.init_weights()
												SCREAMING_SNAKE_CASE    			=       0
												SCREAMING_SNAKE_CASE    			=       0
												SCREAMING_SNAKE_CASE    			=       0

												SCREAMING_SNAKE_CASE    			=       0




							def 	__snake_case(						self   :					int	,							_UpperCamelCase   :					List[Any]      )					->				Dict:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       threshold




							def 	__snake_case(						self   :					Tuple	,							_UpperCamelCase   :					Any      )					->				Dict:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       patience




							def 	__snake_case(						self   :					Tuple      )					->				int:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       0
												SCREAMING_SNAKE_CASE    			=       0




							def 	__snake_case(						self   :					Any      )					->				Dict:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       self.inference_layers_num / self.inference_instances_num
												SCREAMING_SNAKE_CASE    			=       (
												    F"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
												    F" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
												)
												print(_UpperCamelCase      )





							@add_start_docstrings_to_model_forward(_UpperCamelCase      )
							def 	__snake_case(						self   :					Dict	,							_UpperCamelCase   :					Optional[Any]=None	,							_UpperCamelCase   :					int=None	,							_UpperCamelCase   :					Union[str, Any]=None	,							_UpperCamelCase   :					Tuple=None	,							_UpperCamelCase   :					Optional[Any]=None	,							_UpperCamelCase   :					Optional[Any]=None	,							_UpperCamelCase   :					Optional[int]=None	,							_UpperCamelCase   :					str=None	,							_UpperCamelCase   :					Tuple=None	,							_UpperCamelCase   :					Tuple=None	,							_UpperCamelCase   :					int=False	,							)					->				Union[str, Any]:



												'''simple docstring'''

												if input_ids is not None and inputs_embeds is not None:
																	raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"      )
												elif input_ids is not None:
																	SCREAMING_SNAKE_CASE    			=       input_ids.size()
												elif inputs_embeds is not None:
																	SCREAMING_SNAKE_CASE    			=       inputs_embeds.size()[:-1]
												else:
																	raise ValueError("You have to specify either input_ids or inputs_embeds"      )

												SCREAMING_SNAKE_CASE    			=       input_ids.device if input_ids is not None else inputs_embeds.device

												if attention_mask is None:
																	SCREAMING_SNAKE_CASE    			=       torch.ones(_UpperCamelCase	,							device=_UpperCamelCase      )
												if token_type_ids is None:
																	SCREAMING_SNAKE_CASE    			=       torch.zeros(_UpperCamelCase	,							dtype=torch.long	,							device=_UpperCamelCase      )

												# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
												# ourselves in which case we just need to make it broadcastable to all heads.
												SCREAMING_SNAKE_CASE    			=       self.get_extended_attention_mask(_UpperCamelCase	,							_UpperCamelCase	,							_UpperCamelCase      )

												# If a 2D ou 3D attention mask is provided for the cross-attention
												# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
												if self.config.is_decoder and encoder_hidden_states is not None:
																	SCREAMING_SNAKE_CASE			,   SCREAMING_SNAKE_CASE			,   SCREAMING_SNAKE_CASE    			=       encoder_hidden_states.size()
																	SCREAMING_SNAKE_CASE    			=       (encoder_batch_size, encoder_sequence_length)
																	if encoder_attention_mask is None:
																						SCREAMING_SNAKE_CASE    			=       torch.ones(_UpperCamelCase	,							device=_UpperCamelCase      )
																	SCREAMING_SNAKE_CASE    			=       self.invert_attention_mask(_UpperCamelCase      )
												else:
																	SCREAMING_SNAKE_CASE    			=       None

												# Prepare head mask if needed
												# 1.0 in head_mask indicate we keep the head
												# attention_probs has shape bsz x n_heads x N x N
												# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
												# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
												SCREAMING_SNAKE_CASE    			=       self.get_head_mask(_UpperCamelCase	,							self.config.num_hidden_layers      )

												SCREAMING_SNAKE_CASE    			=       self.embeddings(
												    input_ids=_UpperCamelCase	,							position_ids=_UpperCamelCase	,							token_type_ids=_UpperCamelCase	,							inputs_embeds=_UpperCamelCase      )
												SCREAMING_SNAKE_CASE    			=       embedding_output

												if self.training:
																	SCREAMING_SNAKE_CASE    			=       []
																	for i in range(self.config.num_hidden_layers      ):
																						SCREAMING_SNAKE_CASE    			=       self.encoder.adaptive_forward(
																						    _UpperCamelCase	,							current_layer=_UpperCamelCase	,							attention_mask=_UpperCamelCase	,							head_mask=_UpperCamelCase      )

																						SCREAMING_SNAKE_CASE    			=       self.pooler(_UpperCamelCase      )
																						SCREAMING_SNAKE_CASE    			=       output_layers[i](output_dropout(_UpperCamelCase      )      )
																						res.append(_UpperCamelCase      )
												elif self.patience == 0:  # Use all layers for inference
																	SCREAMING_SNAKE_CASE    			=       self.encoder(
																	    _UpperCamelCase	,							attention_mask=_UpperCamelCase	,							head_mask=_UpperCamelCase	,							encoder_hidden_states=_UpperCamelCase	,							encoder_attention_mask=_UpperCamelCase	,							)
																	SCREAMING_SNAKE_CASE    			=       self.pooler(encoder_outputs[0]      )
																	SCREAMING_SNAKE_CASE    			=       [output_layers[self.config.num_hidden_layers - 1](_UpperCamelCase      )]
												else:
																	SCREAMING_SNAKE_CASE    			=       0
																	SCREAMING_SNAKE_CASE    			=       None
																	SCREAMING_SNAKE_CASE    			=       0
																	for i in range(self.config.num_hidden_layers      ):
																						calculated_layer_num += 1
																						SCREAMING_SNAKE_CASE    			=       self.encoder.adaptive_forward(
																						    _UpperCamelCase	,							current_layer=_UpperCamelCase	,							attention_mask=_UpperCamelCase	,							head_mask=_UpperCamelCase      )

																						SCREAMING_SNAKE_CASE    			=       self.pooler(_UpperCamelCase      )
																						SCREAMING_SNAKE_CASE    			=       output_layers[i](_UpperCamelCase      )
																						if regression:
																											SCREAMING_SNAKE_CASE    			=       logits.detach()
																											if patient_result is not None:
																																SCREAMING_SNAKE_CASE    			=       patient_result.detach()
																											if (patient_result is not None) and torch.abs(patient_result - labels      ) < self.regression_threshold:
																																patient_counter += 1
																											else:
																																SCREAMING_SNAKE_CASE    			=       0
																						else:
																											SCREAMING_SNAKE_CASE    			=       logits.detach().argmax(dim=1      )
																											if patient_result is not None:
																																SCREAMING_SNAKE_CASE    			=       patient_result.detach().argmax(dim=1      )
																											if (patient_result is not None) and torch.all(labels.eq(_UpperCamelCase      )      ):
																																patient_counter += 1
																											else:
																																SCREAMING_SNAKE_CASE    			=       0

																						SCREAMING_SNAKE_CASE    			=       logits
																						if patient_counter == self.patience:
																											break
																	SCREAMING_SNAKE_CASE    			=       [patient_result]
																	self.inference_layers_num += calculated_layer_num
																	self.inference_instances_num += 1

												return res



@add_start_docstrings(
    """Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. """				,       a				,       )
class 						lowercase    (     a     ):




							def __init__(						self   :					Any	,							_UpperCamelCase   :					str      )					->				Optional[int]:



												'''simple docstring'''

												super().__init__(_UpperCamelCase      )
												SCREAMING_SNAKE_CASE    			=       config.num_labels

												SCREAMING_SNAKE_CASE    			=       BertModelWithPabee(_UpperCamelCase      )
												SCREAMING_SNAKE_CASE    			=       nn.Dropout(config.hidden_dropout_prob      )
												SCREAMING_SNAKE_CASE    			=       nn.ModuleList(
												    [nn.Linear(config.hidden_size	,							self.config.num_labels      ) for _ in range(config.num_hidden_layers      )]      )

												self.init_weights()





							@add_start_docstrings_to_model_forward(_UpperCamelCase      )
							def 	__snake_case(						self   :					str	,							_UpperCamelCase   :					Optional[int]=None	,							_UpperCamelCase   :					Dict=None	,							_UpperCamelCase   :					Optional[Any]=None	,							_UpperCamelCase   :					Any=None	,							_UpperCamelCase   :					Optional[Any]=None	,							_UpperCamelCase   :					Dict=None	,							_UpperCamelCase   :					int=None	,							)					->				List[str]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       self.bert(
												    input_ids=_UpperCamelCase	,							attention_mask=_UpperCamelCase	,							token_type_ids=_UpperCamelCase	,							position_ids=_UpperCamelCase	,							head_mask=_UpperCamelCase	,							inputs_embeds=_UpperCamelCase	,							output_dropout=self.dropout	,							output_layers=self.classifiers	,							regression=self.num_labels == 1	,							)

												SCREAMING_SNAKE_CASE    			=       (logits[-1],)

												if labels is not None:
																	SCREAMING_SNAKE_CASE    			=       None
																	SCREAMING_SNAKE_CASE    			=       0
																	for ix, logits_item in enumerate(_UpperCamelCase      ):
																						if self.num_labels == 1:
																											#  We are doing regression
																											SCREAMING_SNAKE_CASE    			=       MSELoss()
																											SCREAMING_SNAKE_CASE    			=       loss_fct(logits_item.view(-1      )	,							labels.view(-1      )      )
																						else:
																											SCREAMING_SNAKE_CASE    			=       CrossEntropyLoss()
																											SCREAMING_SNAKE_CASE    			=       loss_fct(logits_item.view(-1	,							self.num_labels      )	,							labels.view(-1      )      )
																						if total_loss is None:
																											SCREAMING_SNAKE_CASE    			=       loss
																						else:
																											total_loss += loss * (ix + 1)
																						total_weights += ix + 1
																	SCREAMING_SNAKE_CASE    			=       (total_loss / total_weights,) + outputs

												return outputs





import collections
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase					:   Dict							=		logging.get_logger(__name__)

_lowerCamelCase					:   List[Any]							=		'''▁'''

_lowerCamelCase					:   Optional[int]							=		{'''vocab_file''': '''prophetnet.tokenizer'''}

_lowerCamelCase					:   str							=		{
    '''vocab_file''': {
        '''microsoft/xprophetnet-large-wiki100-cased''': (
            '''https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/prophetnet.tokenizer'''
        ),
    }
}

_lowerCamelCase					:   Optional[Any]							=		{
    '''microsoft/xprophetnet-large-wiki100-cased''': {'''do_lower_case''': False},
}

_lowerCamelCase					:   Optional[Any]							=		{
    '''microsoft/xprophetnet-large-wiki100-cased''': 5_12,
}





def 	__lowerCamelCase							(UpperCAmelCase__     :							Optional[Any]  ):
					SCREAMING_SNAKE_CASE    			=       collections.OrderedDict()
					with open(UpperCAmelCase__      , "r"      , encoding="utf-8"  ) as reader:
										SCREAMING_SNAKE_CASE    			=       reader.readlines()
					for index, token in enumerate(UpperCAmelCase__  ):
										SCREAMING_SNAKE_CASE    			=       token.rstrip("\n"  )
										SCREAMING_SNAKE_CASE    			=       index
					return vocab


class 						lowercase    (     a     ):
							lowercase__						:       Optional[int]								=							VOCAB_FILES_NAMES
							lowercase__						:       Optional[Any]								=							PRETRAINED_VOCAB_FILES_MAP
							lowercase__						:       Union[str, Any]								=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase__						:       Any								=							["""input_ids""", """attention_mask"""]




							def __init__(						self   :					Optional[int]	,							_UpperCamelCase   :					List[str]	,							_UpperCamelCase   :					str="[SEP]"	,							_UpperCamelCase   :					str="[SEP]"	,							_UpperCamelCase   :					Dict="[SEP]"	,							_UpperCamelCase   :					Tuple="[UNK]"	,							_UpperCamelCase   :					Dict="[PAD]"	,							_UpperCamelCase   :					Any="[CLS]"	,							_UpperCamelCase   :					Optional[Any]="[MASK]"	,							_UpperCamelCase   :					Optional[Dict[str, Any]] = None	,							**_UpperCamelCase   :					Dict	,							)					->				None:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       {} if sp_model_kwargs is None else sp_model_kwargs

												super().__init__(
												    bos_token=_UpperCamelCase	,							eos_token=_UpperCamelCase	,							sep_token=_UpperCamelCase	,							unk_token=_UpperCamelCase	,							pad_token=_UpperCamelCase	,							cls_token=_UpperCamelCase	,							mask_token=_UpperCamelCase	,							sp_model_kwargs=self.sp_model_kwargs	,							**_UpperCamelCase	,							)

												try:
																	import sentencepiece as spm
												except ImportError:
																	logger.warning(
																	    "You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece"
																	    " pip install sentencepiece"      )
																	raise

												SCREAMING_SNAKE_CASE    			=       spm.SentencePieceProcessor(**self.sp_model_kwargs      )
												self.sp_model.Load(str(_UpperCamelCase      )      )
												SCREAMING_SNAKE_CASE    			=       vocab_file

												# Original fairseq vocab and spm vocab must be "aligned":
												# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
												# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
												# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
												# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

												# put special tokens and [unused] tokens into the vocab
												SCREAMING_SNAKE_CASE    			=       {"[PAD]": 0, "[CLS]": 1, "[SEP]": 2, "[UNK]": 3, "[MASK]": 4}

												for i in range(10      ):
																	SCREAMING_SNAKE_CASE    			=       F"[unused{i}]"
																	SCREAMING_SNAKE_CASE    			=       5 + i

												# The first "real" token "," has position 15 in the embedding vocab and position 3 in the spm vocab
												SCREAMING_SNAKE_CASE    			=       12
												SCREAMING_SNAKE_CASE    			=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}
												for k in self.fairseq_tokens_to_ids.keys():
																	self.unique_no_split_tokens.append(_UpperCamelCase      )




							def __getstate__(						self   :					Dict      )					->				str:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       self.__dict__.copy()
												SCREAMING_SNAKE_CASE    			=       None
												return state




							def __setstate__(						self   :					List[Any]	,							_UpperCamelCase   :					Union[str, Any]      )					->				Dict:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       d
												try:
																	import sentencepiece as spm
												except ImportError:
																	logger.warning(
																	    "You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece"
																	    " pip install sentencepiece"      )
																	raise

												# for backward compatibility
												if not hasattr(self	,							"sp_model_kwargs"      ):
																	SCREAMING_SNAKE_CASE    			=       {}

												SCREAMING_SNAKE_CASE    			=       spm.SentencePieceProcessor(**self.sp_model_kwargs      )
												self.sp_model.Load(self.vocab_file      )




							def 	__snake_case(						self   :					Dict	,							_UpperCamelCase   :					List[int]	,							_UpperCamelCase   :					Optional[List[int]] = None	,							_UpperCamelCase   :					bool = False      )					->				List[int]:



												'''simple docstring'''

												if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=_UpperCamelCase	,							token_ids_a=_UpperCamelCase	,							already_has_special_tokens=_UpperCamelCase      )

												if token_ids_a is None:
																	return ([0] * len(_UpperCamelCase      )) + [1]
												return ([0] * len(_UpperCamelCase      )) + [1] + ([0] * len(_UpperCamelCase      )) + [1]




							def 	__snake_case(						self   :					str	,							_UpperCamelCase   :					List[int]	,							_UpperCamelCase   :					Optional[List[int]] = None      )					->				List[int]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       [self.sep_token_id]

												if token_ids_a is None:
																	return len(token_ids_a + sep      ) * [0]
												return len(token_ids_a + sep + sep + token_ids_a + sep      ) * [0]




							@property
							def 	__snake_case(						self   :					Dict      )					->				Optional[Any]:



												'''simple docstring'''

												return len(self.sp_model      ) + self.fairseq_offset




							def 	__snake_case(						self   :					Optional[Any]      )					->				Union[str, Any]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       {self.convert_ids_to_tokens(_UpperCamelCase      ): i for i in range(self.vocab_size      )}
												vocab.update(self.added_tokens_encoder      )
												return vocab




							def 	__snake_case(						self   :					Union[str, Any]	,							_UpperCamelCase   :					str      )					->				str:



												'''simple docstring'''

												return self.sp_model.encode(_UpperCamelCase	,							out_type=_UpperCamelCase      )




							def 	__snake_case(						self   :					Optional[Any]	,							_UpperCamelCase   :					List[Any]      )					->				List[str]:



												'''simple docstring'''

												if token in self.fairseq_tokens_to_ids:
																	return self.fairseq_tokens_to_ids[token]
												SCREAMING_SNAKE_CASE    			=       self.sp_model.PieceToId(_UpperCamelCase      )

												# Need to return unknown token if the SP model returned 0
												return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




							def 	__snake_case(						self   :					str	,							_UpperCamelCase   :					str      )					->				int:



												'''simple docstring'''

												if index in self.fairseq_ids_to_tokens:
																	return self.fairseq_ids_to_tokens[index]
												return self.sp_model.IdToPiece(index - self.fairseq_offset      )




							def 	__snake_case(						self   :					List[str]	,							_UpperCamelCase   :					Dict      )					->				Optional[Any]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       "".join(_UpperCamelCase      ).replace(_UpperCamelCase	,							" "      ).strip()
												return out_string




							def 	__snake_case(						self   :					Optional[Any]	,							_UpperCamelCase   :					str	,							_UpperCamelCase   :					Optional[str] = None      )					->				Tuple[str]:



												'''simple docstring'''

												if not os.path.isdir(_UpperCamelCase      ):
																	logger.error(F"Vocabulary path ({save_directory}) should be a directory"      )
																	return
												SCREAMING_SNAKE_CASE    			=       os.path.join(
												    _UpperCamelCase	,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )

												if os.path.abspath(self.vocab_file      ) != os.path.abspath(_UpperCamelCase      ) and os.path.isfile(self.vocab_file      ):
																	copyfile(self.vocab_file	,							_UpperCamelCase      )
												elif not os.path.isfile(self.vocab_file      ):
																	with open(_UpperCamelCase	,							"wb"      ) as fi:
																						SCREAMING_SNAKE_CASE    			=       self.sp_model.serialized_model_proto()
																						fi.write(_UpperCamelCase      )

												return (out_vocab_file,)





							def 	__snake_case(						self   :					Optional[Any]	,							_UpperCamelCase   :					List[int]	,							_UpperCamelCase   :					Optional[List[int]] = None      )					->				List[int]:



												'''simple docstring'''

												if token_ids_a is None:
																	return token_ids_a + [self.sep_token_id]
												SCREAMING_SNAKE_CASE    			=       [self.sep_token_id]
												return token_ids_a + sep + token_ids_a + sep




