
"""simple docstring"""




from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers import (
        TFConvBertForMaskedLM,
        TFConvBertForMultipleChoice,
        TFConvBertForQuestionAnswering,
        TFConvBertForSequenceClassification,
        TFConvBertForTokenClassification,
        TFConvBertModel,
    )







class   _lowercase						:


     '''simple docstring'''





     def __init__(      self ,			__UpperCamelCase ,			__UpperCamelCase=13 ,			__UpperCamelCase=7 ,			__UpperCamelCase=True ,			__UpperCamelCase=True ,			__UpperCamelCase=True ,			__UpperCamelCase=True ,			__UpperCamelCase=99 ,			__UpperCamelCase=32 ,			__UpperCamelCase=2 ,			__UpperCamelCase=4 ,			__UpperCamelCase=37 ,			__UpperCamelCase="gelu" ,			__UpperCamelCase=0.1 ,			__UpperCamelCase=0.1 ,			__UpperCamelCase=5_12 ,			__UpperCamelCase=16 ,			__UpperCamelCase=2 ,			__UpperCamelCase=0.02 ,			__UpperCamelCase=3 ,			__UpperCamelCase=4 ,			__UpperCamelCase=None ,			)->					List[str]:

          '''simple docstring'''





          UpperCAmelCase__							: Tuple 							=						parent
          UpperCAmelCase__							: int 							=						13
          UpperCAmelCase__							: Union[str, Any] 							=						7
          UpperCAmelCase__							: Tuple 							=						True
          UpperCAmelCase__							: Dict 							=						True
          UpperCAmelCase__							: Optional[int] 							=						True
          UpperCAmelCase__							: Union[str, Any] 							=						True
          UpperCAmelCase__							: Union[str, Any] 							=						99
          UpperCAmelCase__							: Any 							=						3_84
          UpperCAmelCase__							: Optional[int] 							=						2
          UpperCAmelCase__							: Tuple 							=						4
          UpperCAmelCase__							: List[str] 							=						37
          UpperCAmelCase__							: Union[str, Any] 							=						"gelu"
          UpperCAmelCase__							: Dict 							=						0.1
          UpperCAmelCase__							: Optional[int] 							=						0.1
          UpperCAmelCase__							: Optional[int] 							=						5_12
          UpperCAmelCase__							: List[str] 							=						16
          UpperCAmelCase__							: Union[str, Any] 							=						2
          UpperCAmelCase__							: List[str] 							=						0.02
          UpperCAmelCase__							: Any 							=						3
          UpperCAmelCase__							: Dict 							=						4
          UpperCAmelCase__							: str 							=						1_28
          UpperCAmelCase__							: List[str] 							=						2
          UpperCAmelCase__							: List[str] 							=						9
          UpperCAmelCase__							: Optional[Any] 							=						1
          UpperCAmelCase__							: Any 							=						None



     def  lowerCAmelCase__       (      self  )->					Tuple:

          '''simple docstring'''





          UpperCAmelCase__							: Optional[int] 							=						ids_tensor([self.batch_size, self.seq_length] ,			self.vocab_size  )

          UpperCAmelCase__							: Union[str, Any] 							=						None
          if self.use_input_mask:
               UpperCAmelCase__							: Dict 							=						random_attention_mask([self.batch_size, self.seq_length]  )

          UpperCAmelCase__							: Any 							=						None
          if self.use_token_type_ids:
               UpperCAmelCase__							: Optional[Any] 							=						ids_tensor([self.batch_size, self.seq_length] ,			self.type_vocab_size  )

          UpperCAmelCase__							: Dict 							=						None
          UpperCAmelCase__							: Union[str, Any] 							=						None
          UpperCAmelCase__							: str 							=						None
          if self.use_labels:
               UpperCAmelCase__							: List[str] 							=						ids_tensor([self.batch_size] ,			self.type_sequence_label_size  )
               UpperCAmelCase__							: List[Any] 							=						ids_tensor([self.batch_size, self.seq_length] ,			self.num_labels  )
               UpperCAmelCase__							: Tuple 							=						ids_tensor([self.batch_size] ,			self.num_choices  )

          UpperCAmelCase__							: List[str] 							=						ConvBertConfig(
              vocab_size=self.vocab_size ,			hidden_size=self.hidden_size ,			num_hidden_layers=self.num_hidden_layers ,			num_attention_heads=self.num_attention_heads ,			intermediate_size=self.intermediate_size ,			hidden_act=self.hidden_act ,			hidden_dropout_prob=self.hidden_dropout_prob ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob ,			max_position_embeddings=self.max_position_embeddings ,			type_vocab_size=self.type_vocab_size ,			initializer_range=self.initializer_range ,			return_dict=__UpperCamelCase ,			)

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )->					Union[str, Any]:

          '''simple docstring'''





          UpperCAmelCase__							: Dict 							=						TFConvBertModel(config=__UpperCamelCase  )
          UpperCAmelCase__							: Optional[int] 							=						{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}

          UpperCAmelCase__							: str 							=						[input_ids, input_mask]
          UpperCAmelCase__							: str 							=						model(__UpperCamelCase  )

          UpperCAmelCase__							: str 							=						model(__UpperCamelCase  )

          self.parent.assertEqual(result.last_hidden_state.shape ,			(self.batch_size, self.seq_length, self.hidden_size)  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )->					Optional[int]:

          '''simple docstring'''





          UpperCAmelCase__							: Dict 							=						TFConvBertForMaskedLM(config=__UpperCamelCase  )
          UpperCAmelCase__							: str 							=						{
              "input_ids": input_ids,
              "attention_mask": input_mask,
              "token_type_ids": token_type_ids,
          }
          UpperCAmelCase__							: Any 							=						model(__UpperCamelCase  )
          self.parent.assertEqual(result.logits.shape ,			(self.batch_size, self.seq_length, self.vocab_size)  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )->					str:

          '''simple docstring'''





          UpperCAmelCase__							: Optional[Any] 							=						self.num_labels
          UpperCAmelCase__							: Tuple 							=						TFConvBertForSequenceClassification(config=__UpperCamelCase  )
          UpperCAmelCase__							: Dict 							=						{
              "input_ids": input_ids,
              "attention_mask": input_mask,
              "token_type_ids": token_type_ids,
          }

          UpperCAmelCase__							: Tuple 							=						model(__UpperCamelCase  )
          self.parent.assertEqual(result.logits.shape ,			(self.batch_size, self.num_labels)  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )->					List[Any]:

          '''simple docstring'''





          UpperCAmelCase__							: List[str] 							=						self.num_choices
          UpperCAmelCase__							: str 							=						TFConvBertForMultipleChoice(config=__UpperCamelCase  )
          UpperCAmelCase__							: Any 							=						tf.tile(tf.expand_dims(__UpperCamelCase ,			1  ) ,			(1, self.num_choices, 1)  )
          UpperCAmelCase__							: List[str] 							=						tf.tile(tf.expand_dims(__UpperCamelCase ,			1  ) ,			(1, self.num_choices, 1)  )
          UpperCAmelCase__							: List[Any] 							=						tf.tile(tf.expand_dims(__UpperCamelCase ,			1  ) ,			(1, self.num_choices, 1)  )
          UpperCAmelCase__							: Optional[Any] 							=						{
              "input_ids": multiple_choice_inputs_ids,
              "attention_mask": multiple_choice_input_mask,
              "token_type_ids": multiple_choice_token_type_ids,
          }
          UpperCAmelCase__							: Dict 							=						model(__UpperCamelCase  )
          self.parent.assertEqual(result.logits.shape ,			(self.batch_size, self.num_choices)  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )->					int:

          '''simple docstring'''





          UpperCAmelCase__							: List[str] 							=						self.num_labels
          UpperCAmelCase__							: Dict 							=						TFConvBertForTokenClassification(config=__UpperCamelCase  )
          UpperCAmelCase__							: Any 							=						{
              "input_ids": input_ids,
              "attention_mask": input_mask,
              "token_type_ids": token_type_ids,
          }
          UpperCAmelCase__							: List[str] 							=						model(__UpperCamelCase  )
          self.parent.assertEqual(result.logits.shape ,			(self.batch_size, self.seq_length, self.num_labels)  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase  )->					List[str]:

          '''simple docstring'''





          UpperCAmelCase__							: int 							=						TFConvBertForQuestionAnswering(config=__UpperCamelCase  )
          UpperCAmelCase__							: Tuple 							=						{
              "input_ids": input_ids,
              "attention_mask": input_mask,
              "token_type_ids": token_type_ids,
          }

          UpperCAmelCase__							: Tuple 							=						model(__UpperCamelCase  )
          self.parent.assertEqual(result.start_logits.shape ,			(self.batch_size, self.seq_length)  )
          self.parent.assertEqual(result.end_logits.shape ,			(self.batch_size, self.seq_length)  )



     def  lowerCAmelCase__       (      self  )->					Union[str, Any]:

          '''simple docstring'''





          UpperCAmelCase__							: Any 							=						self.prepare_config_and_inputs()
          (
              UpperCAmelCase__	
          )							: Dict 							=						config_and_inputs
          UpperCAmelCase__							: Optional[int] 							=						{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
          return config, inputs_dict







@require_tf
class   _lowercase						(       lowerCAmelCase_	,   lowerCAmelCase_	,   unittest.TestCase				):


     '''simple docstring'''





     _A								=							(
         (
             TFConvBertModel,
             TFConvBertForMaskedLM,
             TFConvBertForQuestionAnswering,
             TFConvBertForSequenceClassification,
             TFConvBertForTokenClassification,
             TFConvBertForMultipleChoice,
         )
         if is_tf_available()
         else ()
     )
     _A								=							(
         {
             'feature-extraction': TFConvBertModel,
             'fill-mask': TFConvBertForMaskedLM,
             'question-answering': TFConvBertForQuestionAnswering,
             'text-classification': TFConvBertForSequenceClassification,
             'token-classification': TFConvBertForTokenClassification,
             'zero-shot': TFConvBertForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     _A								=							False
     _A								=							False
     _A								=							False



     def  lowerCAmelCase__       (      self  )->					List[str]:

          '''simple docstring'''





          UpperCAmelCase__							: Optional[int] 							=						TFConvBertModelTester(self  )
          UpperCAmelCase__							: Union[str, Any] 							=						ConfigTester(self ,			config_class=__UpperCamelCase ,			hidden_size=37  )



     def  lowerCAmelCase__       (      self  )->					Tuple:

          '''simple docstring'''





          self.config_tester.run_common_tests()



     def  lowerCAmelCase__       (      self  )->					int:

          '''simple docstring'''





          UpperCAmelCase__							: Optional[Any] 							=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*__UpperCamelCase  )



     def  lowerCAmelCase__       (      self  )->					str:

          '''simple docstring'''





          UpperCAmelCase__							: int 							=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*__UpperCamelCase  )



     def  lowerCAmelCase__       (      self  )->					Optional[Any]:

          '''simple docstring'''





          UpperCAmelCase__							: str 							=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*__UpperCamelCase  )



     def  lowerCAmelCase__       (      self  )->					int:

          '''simple docstring'''





          UpperCAmelCase__							: List[str] 							=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*__UpperCamelCase  )



     def  lowerCAmelCase__       (      self  )->					Optional[int]:

          '''simple docstring'''





          UpperCAmelCase__							: str 							=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*__UpperCamelCase  )



     def  lowerCAmelCase__       (      self  )->					int:

          '''simple docstring'''





          UpperCAmelCase__							: Optional[Any] 							=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*__UpperCamelCase  )



     @slow
     def  lowerCAmelCase__       (      self  )->					Dict:

          '''simple docstring'''





          UpperCAmelCase__							: int 							=						self.model_tester.prepare_config_and_inputs_for_common()
          UpperCAmelCase__							: List[Any] 							=						True
          UpperCAmelCase__							: Optional[int] 							=						True

          if hasattr(__UpperCamelCase ,			"use_cache"  ):
               UpperCAmelCase__							: List[Any] 							=						True

          UpperCAmelCase__							: int 							=						getattr(self.model_tester ,			"encoder_seq_length" ,			self.model_tester.seq_length  )
          UpperCAmelCase__							: Any 							=						getattr(self.model_tester ,			"key_length" ,			__UpperCamelCase  )

          for model_class in self.all_model_classes:
               UpperCAmelCase__							: Tuple 							=						self._prepare_for_class(__UpperCamelCase ,			__UpperCamelCase  )
               UpperCAmelCase__							: Union[str, Any] 							=						model_class(__UpperCamelCase  )
               UpperCAmelCase__							: str 							=						len(model(__UpperCamelCase  )  )

               with tempfile.TemporaryDirectory() as tmpdirname:
                    model.save_pretrained(__UpperCamelCase ,			saved_model=__UpperCamelCase  )
                    UpperCAmelCase__							: Union[str, Any] 							=						os.path.join(__UpperCamelCase ,			"saved_model" ,			"1"  )
                    UpperCAmelCase__							: int 							=						tf.keras.models.load_model(__UpperCamelCase  )
                    UpperCAmelCase__							: List[Any] 							=						model(__UpperCamelCase  )

                    if self.is_encoder_decoder:
                         UpperCAmelCase__							: Optional[int] 							=						outputs["encoder_hidden_states"]
                         UpperCAmelCase__							: Tuple 							=						outputs["encoder_attentions"]
                    else:
                         UpperCAmelCase__							: List[Any] 							=						outputs["hidden_states"]
                         UpperCAmelCase__							: Union[str, Any] 							=						outputs["attentions"]

                    self.assertEqual(len(__UpperCamelCase  ) ,			__UpperCamelCase  )

                    UpperCAmelCase__							: Optional[int] 							=						getattr(
                        self.model_tester ,			"expected_num_hidden_layers" ,			self.model_tester.num_hidden_layers + 1  )

                    self.assertEqual(len(__UpperCamelCase  ) ,			__UpperCamelCase  )
                    self.assertListEqual(
                        list(output_hidden_states[0].shape[-2:]  ) ,			[self.model_tester.seq_length, self.model_tester.hidden_size] ,			)

                    self.assertEqual(len(__UpperCamelCase  ) ,			self.model_tester.num_hidden_layers  )
                    self.assertListEqual(
                        list(output_attentions[0].shape[-3:]  ) ,			[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length] ,			)



     @slow
     def  lowerCAmelCase__       (      self  )->					Union[str, Any]:

          '''simple docstring'''





          UpperCAmelCase__							: Tuple 							=						TFConvBertModel.from_pretrained("YituTech/conv-bert-base"  )
          self.assertIsNotNone(__UpperCamelCase  )



     def  lowerCAmelCase__       (      self  )->					Union[str, Any]:

          '''simple docstring'''





          UpperCAmelCase__							: Optional[int] 							=						self.model_tester.prepare_config_and_inputs_for_common()
          UpperCAmelCase__							: Optional[int] 							=						True
          UpperCAmelCase__							: List[Any] 							=						getattr(self.model_tester ,			"decoder_seq_length" ,			self.model_tester.seq_length  )
          UpperCAmelCase__							: int 							=						getattr(self.model_tester ,			"encoder_seq_length" ,			self.model_tester.seq_length  )
          UpperCAmelCase__							: Union[str, Any] 							=						getattr(self.model_tester ,			"key_length" ,			__UpperCamelCase  )
          UpperCAmelCase__							: Any 							=						getattr(self.model_tester ,			"key_length" ,			__UpperCamelCase  )

          def check_decoder_attentions_output(__UpperCamelCase  ):
               UpperCAmelCase__							: str 							=						len(__UpperCamelCase  )
               self.assertEqual(out_len % 2 ,			0  )
               UpperCAmelCase__							: Union[str, Any] 							=						outputs.decoder_attentions
               self.assertEqual(len(__UpperCamelCase  ) ,			self.model_tester.num_hidden_layers  )
               self.assertListEqual(
                   list(decoder_attentions[0].shape[-3:]  ) ,			[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length] ,			)

          def check_encoder_attentions_output(__UpperCamelCase  ):
               UpperCAmelCase__							: Optional[Any] 							=						[
                   t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
               ]
               self.assertEqual(len(__UpperCamelCase  ) ,			self.model_tester.num_hidden_layers  )
               self.assertListEqual(
                   list(attentions[0].shape[-3:]  ) ,			[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length] ,			)

          for model_class in self.all_model_classes:
               UpperCAmelCase__							: List[Any] 							=						True
               UpperCAmelCase__							: List[Any] 							=						False
               UpperCAmelCase__							: Optional[int] 							=						model_class(__UpperCamelCase  )
               UpperCAmelCase__							: Union[str, Any] 							=						model(self._prepare_for_class(__UpperCamelCase ,			__UpperCamelCase  )  )
               UpperCAmelCase__							: Optional[Any] 							=						len(__UpperCamelCase  )
               self.assertEqual(config.output_hidden_states ,			__UpperCamelCase  )
               check_encoder_attentions_output(__UpperCamelCase  )

               if self.is_encoder_decoder:
                    UpperCAmelCase__							: List[str] 							=						model_class(__UpperCamelCase  )
                    UpperCAmelCase__							: List[Any] 							=						model(self._prepare_for_class(__UpperCamelCase ,			__UpperCamelCase  )  )
                    self.assertEqual(config.output_hidden_states ,			__UpperCamelCase  )
                    check_decoder_attentions_output(__UpperCamelCase  )

               # Check that output attentions can also be changed via the config
               del inputs_dict["output_attentions"]
               UpperCAmelCase__							: Tuple 							=						True
               UpperCAmelCase__							: Optional[Any] 							=						model_class(__UpperCamelCase  )
               UpperCAmelCase__							: Optional[int] 							=						model(self._prepare_for_class(__UpperCamelCase ,			__UpperCamelCase  )  )
               self.assertEqual(config.output_hidden_states ,			__UpperCamelCase  )
               check_encoder_attentions_output(__UpperCamelCase  )

               # Check attention is always last and order is fine
               UpperCAmelCase__							: Union[str, Any] 							=						True
               UpperCAmelCase__							: Tuple 							=						True
               UpperCAmelCase__							: Dict 							=						model_class(__UpperCamelCase  )
               UpperCAmelCase__							: Any 							=						model(self._prepare_for_class(__UpperCamelCase ,			__UpperCamelCase  )  )

               self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1) ,			len(__UpperCamelCase  )  )
               self.assertEqual(model.config.output_hidden_states ,			__UpperCamelCase  )
               check_encoder_attentions_output(__UpperCamelCase  )







@require_tf
class   _lowercase						(       unittest.TestCase				):


     '''simple docstring'''





     @slow
     def  lowerCAmelCase__       (      self  )->					Optional[Any]:

          '''simple docstring'''





          UpperCAmelCase__							: Optional[int] 							=						TFConvBertModel.from_pretrained("YituTech/conv-bert-base"  )
          UpperCAmelCase__							: Union[str, Any] 							=						tf.constant([[0, 1, 2, 3, 4, 5]]  )
          UpperCAmelCase__							: int 							=						model(__UpperCamelCase  )[0]

          UpperCAmelCase__							: Any 							=						[1, 6, 7_68]
          self.assertEqual(output.shape ,			__UpperCamelCase  )

          UpperCAmelCase__							: Optional[int] 							=						tf.constant(
              [
                  [
                      [-0.0347_5493, -0.468_6034, -0.3063_8832],
                      [0.2263_7248, -0.2698_8646, -0.742_3424],
                      [0.1032_4868, -0.4501_3508, -0.5828_0784],
                  ]
              ]  )
          tf.debugging.assert_near(output[:, :3, :3] ,			__UpperCamelCase ,			atol=1E-4  )





"""simple docstring"""




from typing import Optional, Union

import torch
from torch import nn

from ...configuration_utils import ConfigMixin, register_to_config
from ...models.modeling_utils import ModelMixin







class   _lowercase						(       lowerCAmelCase_	,   lowerCAmelCase_				):


     '''simple docstring'''





     @register_to_config
     def __init__(      self ,			__UpperCamelCase = 7_68 ,			)->					Union[str, Any]:
          super().__init__()

          UpperCAmelCase__							: str 							=						nn.Parameter(torch.zeros(1 ,			__UpperCamelCase  )  )
          UpperCAmelCase__							: Optional[int] 							=						nn.Parameter(torch.ones(1 ,			__UpperCamelCase  )  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase = None ,			__UpperCamelCase = None ,			)->					Any:
          UpperCAmelCase__							: Dict 							=						nn.Parameter(self.mean.to(__UpperCamelCase  ).to(__UpperCamelCase  )  )
          UpperCAmelCase__							: Any 							=						nn.Parameter(self.std.to(__UpperCamelCase  ).to(__UpperCamelCase  )  )
          return self



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Union[str, Any]:
          UpperCAmelCase__							: Dict 							=						(embeds - self.mean) * 1.0 / self.std
          return embeds



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					List[Any]:
          UpperCAmelCase__							: Any 							=						(embeds * self.std) + self.mean
          return embeds



