



from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy



@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """  ,	lowerCAmelCase__  ,	)
class  __lowerCAmelCase     (	lowerCAmelCase__   ):
 lowerCAmelCase__       					=  RobertaConfig
 lowerCAmelCase__       					=  """roberta"""




 def __init__(						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    super().__init__(__UpperCAmelCase				)

    __lowerCamelCase													=    RobertaEmbeddings(__UpperCAmelCase				)
    self.init_weights()



@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """  ,	lowerCAmelCase__  ,	)
class  __lowerCAmelCase     (	lowerCAmelCase__   ):
 lowerCAmelCase__       					=  RobertaConfig
 lowerCAmelCase__       					=  """roberta"""




 def __init__(						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    super().__init__(__UpperCAmelCase				)
    __lowerCamelCase													=    config.num_labels
    __lowerCamelCase													=    config.num_hidden_layers

    __lowerCamelCase													=    DeeRobertaModel(__UpperCAmelCase				)
    __lowerCamelCase													=    nn.Dropout(config.hidden_dropout_prob				)
    __lowerCamelCase													=    nn.Linear(config.hidden_size	,				self.config.num_labels				)




 @add_start_docstrings_to_model_forward(__UpperCAmelCase				)
 def        lowerCamelCase      (						self	,				__UpperCAmelCase=None	,				__UpperCAmelCase=None	,				__UpperCAmelCase=None	,				__UpperCAmelCase=None	,				__UpperCAmelCase=None	,				__UpperCAmelCase=None	,				__UpperCAmelCase=None	,				__UpperCAmelCase=-1	,				__UpperCAmelCase=False	,				):



    '''simple docstring'''
    __lowerCamelCase													=    self.num_layers
    try:
       __lowerCamelCase													=    self.roberta(
           __UpperCAmelCase	,				attention_mask=__UpperCAmelCase	,				token_type_ids=__UpperCAmelCase	,				position_ids=__UpperCAmelCase	,				head_mask=__UpperCAmelCase	,				inputs_embeds=__UpperCAmelCase	,				)

       __lowerCamelCase													=    outputs[1]

       __lowerCamelCase													=    self.dropout(__UpperCAmelCase				)
       __lowerCamelCase													=    self.classifier(__UpperCAmelCase				)
       __lowerCamelCase													=    (logits,) + outputs[2:]  # add hidden states and attention if they are here
    except HighwayException as e:
       __lowerCamelCase													=    e.message
       __lowerCamelCase													=    e.exit_layer
       __lowerCamelCase													=    outputs[0]

    if not self.training:
       __lowerCamelCase													=    entropy(__UpperCAmelCase				)
       __lowerCamelCase													=    []
       __lowerCamelCase													=    []
    if labels is not None:
       if self.num_labels == 1:
          #  We are doing regression
          __lowerCamelCase													=    MSELoss()
          __lowerCamelCase													=    loss_fct(logits.view(-1				)	,				labels.view(-1				)				)
       else:
          __lowerCamelCase													=    CrossEntropyLoss()
          __lowerCamelCase													=    loss_fct(logits.view(-1	,				self.num_labels				)	,				labels.view(-1				)				)

       # work with highway exits
       __lowerCamelCase													=    []
       for highway_exit in outputs[-1]:
          __lowerCamelCase													=    highway_exit[0]
          if not self.training:
             highway_logits_all.append(__UpperCAmelCase				)
             highway_entropy.append(highway_exit[2]				)
          if self.num_labels == 1:
             #  We are doing regression
             __lowerCamelCase													=    MSELoss()
             __lowerCamelCase													=    loss_fct(highway_logits.view(-1				)	,				labels.view(-1				)				)
          else:
             __lowerCamelCase													=    CrossEntropyLoss()
             __lowerCamelCase													=    loss_fct(highway_logits.view(-1	,				self.num_labels				)	,				labels.view(-1				)				)
          highway_losses.append(__UpperCAmelCase				)

       if train_highway:
          __lowerCamelCase													=    (sum(highway_losses[:-1]				),) + outputs
          # exclude the final highway, of course
       else:
          __lowerCamelCase													=    (loss,) + outputs
    if not self.training:
       __lowerCamelCase													=    outputs + ((original_entropy, highway_entropy), exit_layer)
       if output_layer >= 0:
          __lowerCamelCase													=    (
              (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
          )  # use the highway of the last layer

    return outputs  # (loss), logits, (hidden_states), (attentions), entropy











import unittest

from transformers import RoFormerTokenizer, RoFormerTokenizerFast
from transformers.testing_utils import require_rjieba, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_rjieba
@require_tokenizers
class  __lowerCAmelCase     (	lowerCAmelCase__  ,	unittest.TestCase   ):
 lowerCAmelCase__       					=  RoFormerTokenizer
 lowerCAmelCase__       					=  RoFormerTokenizerFast
 lowerCAmelCase__       					=  True
 lowerCAmelCase__       					=  True




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    super().setUp()




 def        lowerCamelCase      (						self	,				**__UpperCAmelCase				):



    '''simple docstring'''
    return self.tokenizer_class.from_pretrained('''junnyu/roformer_chinese_base'''	,				**__UpperCAmelCase				)




 def        lowerCamelCase      (						self	,				**__UpperCAmelCase				):



    '''simple docstring'''
    return self.rust_tokenizer_class.from_pretrained('''junnyu/roformer_chinese_base'''	,				**__UpperCAmelCase				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''永和服装饰品有限公司,今天天气非常好'''
    __lowerCamelCase													=    '''永和 服装 饰品 有限公司 , 今 天 天 气 非常 好'''
    return input_text, output_text




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.get_tokenizer()
    __lowerCamelCase							,__lowerCamelCase													=    self.get_chinese_input_output_texts()
    __lowerCamelCase													=    tokenizer.tokenize(__UpperCAmelCase				)

    self.assertListEqual(__UpperCAmelCase	,				output_text.split()				)

    __lowerCamelCase													=    tokens + [tokenizer.unk_token]
    __lowerCamelCase													=    [22943, 21332, 34431, 45904, 117, 306, 1231, 1231, 2653, 33994, 1266, 100]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)	,				__UpperCAmelCase				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.get_rust_tokenizer()
    __lowerCamelCase							,__lowerCamelCase													=    self.get_chinese_input_output_texts()
    __lowerCamelCase													=    tokenizer.tokenize(__UpperCAmelCase				)
    self.assertListEqual(__UpperCAmelCase	,				output_text.split()				)
    __lowerCamelCase													=    tokens + [tokenizer.unk_token]
    __lowerCamelCase													=    [22943, 21332, 34431, 45904, 117, 306, 1231, 1231, 2653, 33994, 1266, 100]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)	,				__UpperCAmelCase				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    pass




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    pass




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    pass






