


import unittest

from transformers import MPNetConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           MPNetForMaskedLM,
           MPNetForMultipleChoice,
           MPNetForQuestionAnswering,
           MPNetForSequenceClassification,
           MPNetForTokenClassification,
           MPNetModel,
       )
class 	UpperCamelCase__					:



     """simple docstring"""






     def __init__( self      ,	snake_case__      ,	snake_case__=13      ,	snake_case__=7      ,	snake_case__=True      ,	snake_case__=True      ,	snake_case__=False      ,	snake_case__=True      ,	snake_case__=99      ,	snake_case__=64      ,	snake_case__=5      ,	snake_case__=4      ,	snake_case__=64      ,	snake_case__="gelu"      ,	snake_case__=0.1      ,	snake_case__=0.1      ,	snake_case__=512      ,	snake_case__=16      ,	snake_case__=2      ,	snake_case__=0.02      ,	snake_case__=3      ,	snake_case__=4      ,	snake_case__=None      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							Union[str, Any]   			=       parent
            _lowerCAmelCase    :							int   			=       batch_size
            _lowerCAmelCase    :							Tuple   			=       seq_length
            _lowerCAmelCase    :							int   			=       is_training
            _lowerCAmelCase    :							str   			=       use_input_mask
            _lowerCAmelCase    :							Dict   			=       use_token_type_ids
            _lowerCAmelCase    :							Union[str, Any]   			=       use_labels
            _lowerCAmelCase    :							List[Any]   			=       vocab_size
            _lowerCAmelCase    :							List[str]   			=       hidden_size
            _lowerCAmelCase    :							Any   			=       num_hidden_layers
            _lowerCAmelCase    :							List[Any]   			=       num_attention_heads
            _lowerCAmelCase    :							List[Any]   			=       intermediate_size
            _lowerCAmelCase    :							Union[str, Any]   			=       hidden_act
            _lowerCAmelCase    :							Union[str, Any]   			=       hidden_dropout_prob
            _lowerCAmelCase    :							List[Any]   			=       attention_probs_dropout_prob
            _lowerCAmelCase    :							Optional[Any]   			=       max_position_embeddings
            _lowerCAmelCase    :							Any   			=       type_vocab_size
            _lowerCAmelCase    :							str   			=       type_sequence_label_size
            _lowerCAmelCase    :							Tuple   			=       initializer_range
            _lowerCAmelCase    :							Optional[int]   			=       num_labels
            _lowerCAmelCase    :							int   			=       num_choices
            _lowerCAmelCase    :							List[Any]   			=       scope







     def a	( self   ):

            '''simple docstring'''

            return MPNetConfig.from_pretrained('microsoft/mpnet-base'   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							List[Any]   			=       ids_tensor([self.batch_size, self.seq_length]      ,	self.vocab_size   )

            _lowerCAmelCase    :							Optional[Any]   			=       None
            if self.use_input_mask:
                   _lowerCAmelCase    :							int   			=       random_attention_mask([self.batch_size, self.seq_length]   )

            _lowerCAmelCase    :							int   			=       None
            _lowerCAmelCase    :							Optional[Any]   			=       None
            _lowerCAmelCase    :							List[Any]   			=       None
            if self.use_labels:
                   _lowerCAmelCase    :							str   			=       ids_tensor([self.batch_size]      ,	self.type_sequence_label_size   )
                   _lowerCAmelCase    :							Optional[Any]   			=       ids_tensor([self.batch_size, self.seq_length]      ,	self.num_labels   )
                   _lowerCAmelCase    :							List[Any]   			=       ids_tensor([self.batch_size]      ,	self.num_choices   )

            _lowerCAmelCase    :							List[Any]   			=       self.get_config()
            return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels







     def a	( self   ):

            '''simple docstring'''

            return MPNetConfig(
                vocab_size=self.vocab_size      ,	hidden_size=self.hidden_size      ,	num_hidden_layers=self.num_hidden_layers      ,	num_attention_heads=self.num_attention_heads      ,	intermediate_size=self.intermediate_size      ,	hidden_act=self.hidden_act      ,	hidden_dropout_prob=self.hidden_dropout_prob      ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,	max_position_embeddings=self.max_position_embeddings      ,	initializer_range=self.initializer_range      ,	)







     def a	( self      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[Any]   			=       MPNetModel(config=snake_case__   )
            model.to(snake_case__   )
            model.eval()
            _lowerCAmelCase    :							List[str]   			=       model(snake_case__      ,	snake_case__   )
            _lowerCAmelCase    :							Optional[Any]   			=       model(snake_case__   )
            self.parent.assertEqual(result.last_hidden_state.shape      ,	(self.batch_size, self.seq_length, self.hidden_size)   )
            self.parent.assertEqual(result.pooler_output.shape      ,	(self.batch_size, self.hidden_size)   )







     def a	( self      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							str   			=       MPNetForQuestionAnswering(config=snake_case__   )
            model.to(snake_case__   )
            model.eval()
            _lowerCAmelCase    :							Optional[int]   			=       model(
                snake_case__      ,	attention_mask=snake_case__      ,	start_positions=snake_case__      ,	end_positions=snake_case__      ,	)
            self.parent.assertEqual(result.start_logits.shape      ,	(self.batch_size, self.seq_length)   )
            self.parent.assertEqual(result.end_logits.shape      ,	(self.batch_size, self.seq_length)   )







     def a	( self      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[Any]   			=       self.num_labels
            _lowerCAmelCase    :							Optional[Any]   			=       MPNetForSequenceClassification(snake_case__   )
            model.to(snake_case__   )
            model.eval()
            _lowerCAmelCase    :							Optional[int]   			=       model(snake_case__      ,	attention_mask=snake_case__      ,	labels=snake_case__   )
            self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, self.num_labels)   )







     def a	( self      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Any   			=       self.num_choices
            _lowerCAmelCase    :							Union[str, Any]   			=       MPNetForMultipleChoice(config=snake_case__   )
            model.to(snake_case__   )
            model.eval()
            _lowerCAmelCase    :							Optional[int]   			=       input_ids.unsqueeze(1   ).expand(-1      ,	self.num_choices      ,	-1   ).contiguous()
            _lowerCAmelCase    :							Any   			=       input_mask.unsqueeze(1   ).expand(-1      ,	self.num_choices      ,	-1   ).contiguous()
            _lowerCAmelCase    :							Optional[Any]   			=       model(
                snake_case__      ,	attention_mask=snake_case__      ,	labels=snake_case__      ,	)
            self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, self.num_choices)   )







     def a	( self      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[int]   			=       self.num_labels
            _lowerCAmelCase    :							str   			=       MPNetForTokenClassification(config=snake_case__   )
            model.to(snake_case__   )
            model.eval()
            _lowerCAmelCase    :							List[str]   			=       model(snake_case__      ,	attention_mask=snake_case__      ,	labels=snake_case__   )
            self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, self.seq_length, self.num_labels)   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							str   			=       self.prepare_config_and_inputs()
            (_lowerCAmelCase)    :							Tuple   			=       config_and_inputs
            _lowerCAmelCase    :							Any   			=       {'input_ids': input_ids, 'attention_mask': input_mask}
            return config, inputs_dict
@require_torch
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_    ,					SCREAMING_SNAKE_CASE_    ,					unittest.TestCase	):



     """simple docstring"""






     __magic_name__    			=    (
         (
             MPNetForMaskedLM,
             MPNetForMultipleChoice,
             MPNetForQuestionAnswering,
             MPNetForSequenceClassification,
             MPNetForTokenClassification,
             MPNetModel,
         )
         if is_torch_available()
         else ()
     )
     __magic_name__    			=    (
         {
             "feature-extraction": MPNetModel,
             "fill-mask": MPNetForMaskedLM,
             "question-answering": MPNetForQuestionAnswering,
             "text-classification": MPNetForSequenceClassification,
             "token-classification": MPNetForTokenClassification,
             "zero-shot": MPNetForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     __magic_name__    			=    False
     __magic_name__    			=    True







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Dict   			=       MPNetModelTester(self   )
            _lowerCAmelCase    :							List[str]   			=       ConfigTester(self      ,	config_class=snake_case__      ,	hidden_size=37   )







     def a	( self   ):

            '''simple docstring'''

            self.config_tester.run_common_tests()







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Tuple   			=       self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_mpnet_model(*snake_case__   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							int   			=       self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_mpnet_for_sequence_classification(*snake_case__   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							str   			=       self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_mpnet_for_multiple_choice(*snake_case__   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Any   			=       self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_mpnet_for_token_classification(*snake_case__   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							List[Any]   			=       self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_mpnet_for_question_answering(*snake_case__   )
@require_torch
class 	UpperCamelCase__					( unittest.TestCase	):



     """simple docstring"""






     @slow
     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Union[str, Any]   			=       MPNetModel.from_pretrained('microsoft/mpnet-base'   )
            _lowerCAmelCase    :							Dict   			=       torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]   )
            _lowerCAmelCase    :							Any   			=       model(snake_case__   )[0]
            _lowerCAmelCase    :							Optional[Any]   			=       torch.Size((1, 11, 768)   )
            self.assertEqual(output.shape      ,	snake_case__   )
            _lowerCAmelCase    :							Dict   			=       torch.tensor(
                [[[-0.0550, 0.1943, -0.0740], [-0.0562, 0.2211, -0.0579], [-0.0437, 0.3337, -0.0641]]]   )
            # compare the actual values for a slice.
            self.assertTrue(torch.allclose(output[:, :3, :3]      ,	snake_case__      ,	atol=1E-4   )   )





'''simple docstring'''







from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowerCAmelCase    :     Optional[Any]					=						logging.get_logger(__name__)

lowerCAmelCase    :     str					=						{
    """EleutherAI/gpt-j-6B""": """https://huggingface.co/EleutherAI/gpt-j-6B/resolve/main/config.json""",
    # See all GPT-J models at https://huggingface.co/models?filter=gpt_j
}
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     __magic_name__    			=    "gptj"
     __magic_name__    			=    {
         "max_position_embeddings": "n_positions",
         "hidden_size": "n_embd",
         "num_attention_heads": "n_head",
         "num_hidden_layers": "n_layer",
     }







     def __init__( self      ,	snake_case__=5_0400      ,	snake_case__=2048      ,	snake_case__=4096      ,	snake_case__=28      ,	snake_case__=16      ,	snake_case__=64      ,	snake_case__=None      ,	snake_case__="gelu_new"      ,	snake_case__=0.0      ,	snake_case__=0.0      ,	snake_case__=0.0      ,	snake_case__=1E-5      ,	snake_case__=0.02      ,	snake_case__=True      ,	snake_case__=5_0256      ,	snake_case__=5_0256      ,	snake_case__=False      ,	**snake_case__      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							int   			=       vocab_size
            _lowerCAmelCase    :							Optional[int]   			=       n_positions
            _lowerCAmelCase    :							Optional[int]   			=       n_embd
            _lowerCAmelCase    :							Optional[int]   			=       n_layer
            _lowerCAmelCase    :							str   			=       n_head
            _lowerCAmelCase    :							Tuple   			=       n_inner
            _lowerCAmelCase    :							Tuple   			=       rotary_dim
            _lowerCAmelCase    :							Optional[int]   			=       activation_function
            _lowerCAmelCase    :							Any   			=       resid_pdrop
            _lowerCAmelCase    :							List[str]   			=       embd_pdrop
            _lowerCAmelCase    :							int   			=       attn_pdrop
            _lowerCAmelCase    :							Any   			=       layer_norm_epsilon
            _lowerCAmelCase    :							Optional[int]   			=       initializer_range
            _lowerCAmelCase    :							List[str]   			=       use_cache

            _lowerCAmelCase    :							Dict   			=       bos_token_id
            _lowerCAmelCase    :							Any   			=       eos_token_id

            super().__init__(
                bos_token_id=snake_case__      ,	eos_token_id=snake_case__      ,	tie_word_embeddings=snake_case__      ,	**snake_case__   )
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     def __init__( self      ,	snake_case__      ,	snake_case__ = "default"      ,	snake_case__ = None      ,	snake_case__ = False      ,	):

            '''simple docstring'''

            super().__init__(snake_case__      ,	task=snake_case__      ,	patching_specs=snake_case__      ,	use_past=snake_case__   )
            if not getattr(self._config      ,	'pad_token_id'      ,	snake_case__   ):
                   # TODO: how to do that better?
                   _lowerCAmelCase    :							Any   			=       0







     @property
     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							str   			=       OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}   )
            if self.use_past:
                   self.fill_with_past_key_values_(snake_case__      ,	direction='inputs'   )
                   _lowerCAmelCase    :							int   			=       {0: 'batch', 1: 'past_sequence + sequence'}
            else:
                   _lowerCAmelCase    :							int   			=       {0: 'batch', 1: 'sequence'}

            return common_inputs







     @property
     def a	( self   ):

            '''simple docstring'''

            return self._config.n_layer







     @property
     def a	( self   ):

            '''simple docstring'''

            return self._config.n_head







     def a	( self      ,	snake_case__      ,	snake_case__ = -1      ,	snake_case__ = -1      ,	snake_case__ = False      ,	snake_case__ = None      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[Any]   			=       super(snake_case__      ,	self   ).generate_dummy_inputs(
                snake_case__      ,	batch_size=snake_case__      ,	seq_length=snake_case__      ,	is_pair=snake_case__      ,	framework=snake_case__   )

            # We need to order the input in the way they appears in the forward()
            _lowerCAmelCase    :							Any   			=       OrderedDict({'input_ids': common_inputs['input_ids']}   )

            # Need to add the past_keys
            if self.use_past:
                   if not is_torch_available():
                          raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'   )
                   else:
                          import torch

                          _lowerCAmelCase			,     _lowerCAmelCase    :							Optional[Any]   			=       common_inputs['input_ids'].shape
                          # Not using the same length for past_key_values
                          _lowerCAmelCase    :							Any   			=       seqlen + 2
                          _lowerCAmelCase    :							Optional[int]   			=       (
                              batch,
                              self.num_attention_heads,
                              past_key_values_length,
                              self._config.hidden_size // self.num_attention_heads,
                          )
                          _lowerCAmelCase    :							Tuple   			=       [
                              (torch.zeros(snake_case__   ), torch.zeros(snake_case__   )) for _ in range(self.num_layers   )
                          ]

            _lowerCAmelCase    :							Tuple   			=       common_inputs['attention_mask']
            if self.use_past:
                   _lowerCAmelCase    :							Any   			=       ordered_inputs['attention_mask'].dtype
                   _lowerCAmelCase    :							Union[str, Any]   			=       torch.cat(
                       [ordered_inputs['attention_mask'], torch.ones(snake_case__      ,	snake_case__      ,	dtype=snake_case__   )]      ,	dim=1   )

            return ordered_inputs







     @property
     def a	( self   ):

            '''simple docstring'''

            return 13

