





import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel







class 			__A       :




      """simple docstring"""







      def __init__(		self      ,							a__      ,							a__=13      ,							a__=7      ,							a__=True      ,							a__=True      ,							a__=False      ,							a__=True      ,							a__=99      ,							a__=32      ,							a__=5      ,							a__=4      ,							a__=37      ,							a__="gelu"      ,							a__=0.1      ,							a__=0.1      ,							a__=512      ,							a__=16      ,							a__=2      ,							a__=0.02      ,							a__=3      ,							a__=4      ,							a__=None      ,							):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		parent
             _lowerCamelCase							:     Optional[Any]	=		batch_size
             _lowerCamelCase							:     List[Any]	=		seq_length
             _lowerCamelCase							:     List[Any]	=		is_training
             _lowerCamelCase							:     Tuple	=		use_input_mask
             _lowerCamelCase							:     Dict	=		use_token_type_ids
             _lowerCamelCase							:     int	=		use_labels
             _lowerCamelCase							:     Dict	=		vocab_size
             _lowerCamelCase							:     Any	=		hidden_size
             _lowerCamelCase							:     List[Any]	=		num_hidden_layers
             _lowerCamelCase							:     str	=		num_attention_heads
             _lowerCamelCase							:     Optional[int]	=		intermediate_size
             _lowerCamelCase							:     Any	=		hidden_act
             _lowerCamelCase							:     Dict	=		hidden_dropout_prob
             _lowerCamelCase							:     str	=		attention_probs_dropout_prob
             _lowerCamelCase							:     int	=		max_position_embeddings
             _lowerCamelCase							:     List[Any]	=		type_vocab_size
             _lowerCamelCase							:     int	=		type_sequence_label_size
             _lowerCamelCase							:     str	=		initializer_range
             _lowerCamelCase							:     str	=		num_labels
             _lowerCamelCase							:     List[Any]	=		num_choices
             _lowerCamelCase							:     str	=		scope


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     int	=		ids_tensor([self.batch_size, self.seq_length]      ,							self.vocab_size)

             _lowerCamelCase							:     List[Any]	=		None
             if self.use_input_mask:
                    _lowerCamelCase							:     Dict	=		random_attention_mask([self.batch_size, self.seq_length])

             _lowerCamelCase							:     Union[str, Any]	=		None
             if self.use_token_type_ids:
                    _lowerCamelCase							:     List[Any]	=		ids_tensor([self.batch_size, self.seq_length]      ,							self.type_vocab_size)

             _lowerCamelCase							:     List[str]	=		None
             _lowerCamelCase							:     Union[str, Any]	=		None
             _lowerCamelCase							:     Optional[int]	=		None
             if self.use_labels:
                    _lowerCamelCase							:     List[Any]	=		ids_tensor([self.batch_size]      ,							self.type_sequence_label_size)
                    _lowerCamelCase							:     str	=		ids_tensor([self.batch_size, self.seq_length]      ,							self.num_labels)
                    _lowerCamelCase							:     Optional[Any]	=		ids_tensor([self.batch_size]      ,							self.num_choices)

             _lowerCamelCase							:     List[str]	=		self.get_config()

             return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


      def 				__snake_case			(		self):



             """simple docstring"""





             return OpenLlamaConfig(
                 vocab_size=self.vocab_size      ,							hidden_size=self.hidden_size      ,							num_hidden_layers=self.num_hidden_layers      ,							num_attention_heads=self.num_attention_heads      ,							intermediate_size=self.intermediate_size      ,							hidden_act=self.hidden_act      ,							hidden_dropout_prob=self.hidden_dropout_prob      ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,							max_position_embeddings=self.max_position_embeddings      ,							type_vocab_size=self.type_vocab_size      ,							is_decoder=a__      ,							initializer_range=self.initializer_range      ,							use_stable_embedding=a__      ,							)


      def 				__snake_case			(		self      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     str	=		OpenLlamaModel(config=a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase							:     Tuple	=		model(a__      ,							attention_mask=a__)
             _lowerCamelCase							:     Union[str, Any]	=		model(a__)
             self.parent.assertEqual(result.last_hidden_state.shape      ,							(self.batch_size, self.seq_length, self.hidden_size))


      def 				__snake_case			(		self      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							):



             """simple docstring"""





             _lowerCamelCase							:     int	=		True
             _lowerCamelCase							:     Optional[Any]	=		OpenLlamaModel(a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase							:     Union[str, Any]	=		model(
                 a__      ,							attention_mask=a__      ,							encoder_hidden_states=a__      ,							encoder_attention_mask=a__      ,							)
             _lowerCamelCase							:     Dict	=		model(
                 a__      ,							attention_mask=a__      ,							encoder_hidden_states=a__      ,							)
             _lowerCamelCase							:     Optional[int]	=		model(a__      ,							attention_mask=a__)
             self.parent.assertEqual(result.last_hidden_state.shape      ,							(self.batch_size, self.seq_length, self.hidden_size))


      def 				__snake_case			(		self      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							):



             """simple docstring"""





             _lowerCamelCase							:     int	=		OpenLlamaForCausalLM(config=a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase							:     str	=		model(a__      ,							attention_mask=a__      ,							labels=a__)
             self.parent.assertEqual(result.logits.shape      ,							(self.batch_size, self.seq_length, self.vocab_size))


      def 				__snake_case			(		self      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							a__      ,							):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		True
             _lowerCamelCase							:     List[Any]	=		True
             _lowerCamelCase							:     List[str]	=		OpenLlamaForCausalLM(config=a__)
             model.to(a__)
             model.eval()

             # first forward pass
             _lowerCamelCase							:     Any	=		model(
                 a__      ,							attention_mask=a__      ,							encoder_hidden_states=a__      ,							encoder_attention_mask=a__      ,							use_cache=a__      ,							)
             _lowerCamelCase							:     Tuple	=		outputs.past_key_values

             # create hypothetical multiple next token and extent to next_input_ids
             _lowerCamelCase							:     str	=		ids_tensor((self.batch_size, 3)      ,							config.vocab_size)
             _lowerCamelCase							:     Tuple	=		ids_tensor((self.batch_size, 3)      ,							vocab_size=2)

             # append to next input_ids and
             _lowerCamelCase							:     Tuple	=		torch.cat([input_ids, next_tokens]      ,							dim=-1)
             _lowerCamelCase							:     List[Any]	=		torch.cat([input_mask, next_mask]      ,							dim=-1)

             _lowerCamelCase							:     Union[str, Any]	=		model(
                 a__      ,							attention_mask=a__      ,							encoder_hidden_states=a__      ,							encoder_attention_mask=a__      ,							output_hidden_states=a__      ,							)['''hidden_states'''][0]
             _lowerCamelCase							:     int	=		model(
                 a__      ,							attention_mask=a__      ,							encoder_hidden_states=a__      ,							encoder_attention_mask=a__      ,							past_key_values=a__      ,							output_hidden_states=a__      ,							)['''hidden_states'''][0]

             # select random slice
             _lowerCamelCase							:     Tuple	=		ids_tensor((1,)      ,							output_from_past.shape[-1]).item()
             _lowerCamelCase							:     Tuple	=		output_from_no_past[:, -3:, random_slice_idx].detach()
             _lowerCamelCase							:     List[str]	=		output_from_past[:, :, random_slice_idx].detach()

             self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

             # test that outputs are equal for slice
             self.parent.assertTrue(torch.allclose(a__      ,							a__      ,							atol=1e-3))



      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		self.prepare_config_and_inputs()
             (
                 _lowerCamelCase  
             )							:     Optional[int]	=		config_and_inputs
             _lowerCamelCase							:     str	=		{'''input_ids''': input_ids, '''attention_mask''': input_mask}
             return config, inputs_dict







@require_torch
class 			__A       (      lowerCamelCase__ ,lowerCamelCase__ ,lowerCamelCase__ ,unittest.TestCase  ):




      """simple docstring"""







      UpperCAmelCase__						=      (
          (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
      )
      UpperCAmelCase__						=      (OpenLlamaForCausalLM,) if is_torch_available() else ()
      UpperCAmelCase__						=      (
          {
              """feature-extraction""": OpenLlamaModel,
              """text-classification""": OpenLlamaForSequenceClassification,
              """text-generation""": OpenLlamaForCausalLM,
              """zero-shot""": OpenLlamaForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      UpperCAmelCase__						=      False
      UpperCAmelCase__						=      False


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		OpenLlamaModelTester(self)
             _lowerCamelCase							:     Any	=		ConfigTester(self      ,							config_class=a__      ,							hidden_size=37)


      def 				__snake_case			(		self):



             """simple docstring"""





             self.config_tester.run_common_tests()


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(*a__)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     List[str]	=		self.model_tester.prepare_config_and_inputs()
             for type in ["absolute", "relative_key", "relative_key_query"]:
                    _lowerCamelCase							:     Union[str, Any]	=		type
                    self.model_tester.create_and_check_model(*a__)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     str	=		self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase							:     int	=		3
             _lowerCamelCase							:     Tuple	=		input_dict['''input_ids''']
             _lowerCamelCase							:     Dict	=		input_ids.ne(1).to(a__)
             _lowerCamelCase							:     List[Any]	=		ids_tensor([self.model_tester.batch_size]      ,							self.model_tester.type_sequence_label_size)
             _lowerCamelCase							:     Optional[int]	=		OpenLlamaForSequenceClassification(a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase							:     Any	=		model(a__      ,							attention_mask=a__      ,							labels=a__)
             self.assertEqual(result.logits.shape      ,							(self.model_tester.batch_size, self.model_tester.num_labels))


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     str	=		self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase							:     Tuple	=		3
             _lowerCamelCase							:     int	=		'''single_label_classification'''
             _lowerCamelCase							:     List[Any]	=		input_dict['''input_ids''']
             _lowerCamelCase							:     str	=		input_ids.ne(1).to(a__)
             _lowerCamelCase							:     List[Any]	=		ids_tensor([self.model_tester.batch_size]      ,							self.model_tester.type_sequence_label_size)
             _lowerCamelCase							:     Optional[int]	=		OpenLlamaForSequenceClassification(a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase							:     Optional[int]	=		model(a__      ,							attention_mask=a__      ,							labels=a__)
             self.assertEqual(result.logits.shape      ,							(self.model_tester.batch_size, self.model_tester.num_labels))


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[Any]	=		self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase							:     str	=		3
             _lowerCamelCase							:     List[Any]	=		'''multi_label_classification'''
             _lowerCamelCase							:     int	=		input_dict['''input_ids''']
             _lowerCamelCase							:     Tuple	=		input_ids.ne(1).to(a__)
             _lowerCamelCase							:     Optional[Any]	=		ids_tensor(
                 [self.model_tester.batch_size, config.num_labels]      ,							self.model_tester.type_sequence_label_size).to(torch.float)
             _lowerCamelCase							:     List[Any]	=		OpenLlamaForSequenceClassification(a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase							:     Dict	=		model(a__      ,							attention_mask=a__      ,							labels=a__)
             self.assertEqual(result.logits.shape      ,							(self.model_tester.batch_size, self.model_tester.num_labels))


      @unittest.skip('''Open-Llama buffers include complex numbers, which breaks this test''')
      def 				__snake_case			(		self):



             """simple docstring"""





             pass



      @parameterized.expand([('''linear''',), ('''dynamic''',)])
      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase							:     Dict	=		ids_tensor([1, 10]      ,							config.vocab_size)
             _lowerCamelCase							:     Dict	=		ids_tensor([1, int(config.max_position_embeddings * 1.5)]      ,							config.vocab_size)

             set_seed(42)  # Fixed seed at init time so the two models get the same random weights
             _lowerCamelCase							:     Union[str, Any]	=		OpenLlamaModel(a__)
             original_model.to(a__)
             original_model.eval()
             _lowerCamelCase							:     Tuple	=		original_model(a__).last_hidden_state
             _lowerCamelCase							:     Dict	=		original_model(a__).last_hidden_state

             set_seed(42)  # Fixed seed at init time so the two models get the same random weights
             _lowerCamelCase							:     str	=		{'''type''': scaling_type, '''factor''': 10.0}
             _lowerCamelCase							:     Union[str, Any]	=		OpenLlamaModel(a__)
             scaled_model.to(a__)
             scaled_model.eval()
             _lowerCamelCase							:     Dict	=		scaled_model(a__).last_hidden_state
             _lowerCamelCase							:     int	=		scaled_model(a__).last_hidden_state

             # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
             # maximum sequence length, so the outputs for the short input should match.
             if scaling_type == "dynamic":
                    self.assertTrue(torch.allclose(a__      ,							a__      ,							atol=1e-5))
             else:
                    self.assertFalse(torch.allclose(a__      ,							a__      ,							atol=1e-5))

             # The output should be different for long inputs
             self.assertFalse(torch.allclose(a__      ,							a__      ,							atol=1e-5))








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


_lowerCamelCase			       =      {
    'configuration_roberta_prelayernorm': [
        'ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'RobertaPreLayerNormConfig',
        'RobertaPreLayerNormOnnxConfig',
    ],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       _lowerCamelCase			       =      [
           'ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST',
           'RobertaPreLayerNormForCausalLM',
           'RobertaPreLayerNormForMaskedLM',
           'RobertaPreLayerNormForMultipleChoice',
           'RobertaPreLayerNormForQuestionAnswering',
           'RobertaPreLayerNormForSequenceClassification',
           'RobertaPreLayerNormForTokenClassification',
           'RobertaPreLayerNormModel',
           'RobertaPreLayerNormPreTrainedModel',
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       _lowerCamelCase			       =      [
           'TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST',
           'TFRobertaPreLayerNormForCausalLM',
           'TFRobertaPreLayerNormForMaskedLM',
           'TFRobertaPreLayerNormForMultipleChoice',
           'TFRobertaPreLayerNormForQuestionAnswering',
           'TFRobertaPreLayerNormForSequenceClassification',
           'TFRobertaPreLayerNormForTokenClassification',
           'TFRobertaPreLayerNormMainLayer',
           'TFRobertaPreLayerNormModel',
           'TFRobertaPreLayerNormPreTrainedModel',
       ]

try:
       if not is_flax_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       _lowerCamelCase			       =      [
           'FlaxRobertaPreLayerNormForCausalLM',
           'FlaxRobertaPreLayerNormForMaskedLM',
           'FlaxRobertaPreLayerNormForMultipleChoice',
           'FlaxRobertaPreLayerNormForQuestionAnswering',
           'FlaxRobertaPreLayerNormForSequenceClassification',
           'FlaxRobertaPreLayerNormForTokenClassification',
           'FlaxRobertaPreLayerNormModel',
           'FlaxRobertaPreLayerNormPreTrainedModel',
       ]


if TYPE_CHECKING:
       from .configuration_roberta_prelayernorm import (
           ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP,
           RobertaPreLayerNormConfig,
           RobertaPreLayerNormOnnxConfig,
       )

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_roberta_prelayernorm import (
                  ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
                  RobertaPreLayerNormForCausalLM,
                  RobertaPreLayerNormForMaskedLM,
                  RobertaPreLayerNormForMultipleChoice,
                  RobertaPreLayerNormForQuestionAnswering,
                  RobertaPreLayerNormForSequenceClassification,
                  RobertaPreLayerNormForTokenClassification,
                  RobertaPreLayerNormModel,
                  RobertaPreLayerNormPreTrainedModel,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_roberta_prelayernorm import (
                  TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFRobertaPreLayerNormForCausalLM,
                  TFRobertaPreLayerNormForMaskedLM,
                  TFRobertaPreLayerNormForMultipleChoice,
                  TFRobertaPreLayerNormForQuestionAnswering,
                  TFRobertaPreLayerNormForSequenceClassification,
                  TFRobertaPreLayerNormForTokenClassification,
                  TFRobertaPreLayerNormMainLayer,
                  TFRobertaPreLayerNormModel,
                  TFRobertaPreLayerNormPreTrainedModel,
              )

       try:
              if not is_flax_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_flax_roberta_prelayernorm import (
                  FlaxRobertaPreLayerNormForCausalLM,
                  FlaxRobertaPreLayerNormForMaskedLM,
                  FlaxRobertaPreLayerNormForMultipleChoice,
                  FlaxRobertaPreLayerNormForQuestionAnswering,
                  FlaxRobertaPreLayerNormForSequenceClassification,
                  FlaxRobertaPreLayerNormForTokenClassification,
                  FlaxRobertaPreLayerNormModel,
                  FlaxRobertaPreLayerNormPreTrainedModel,
              )

else:
       import sys

       _lowerCamelCase			       =      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

