"""simple docstring"""


from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import (
     TFConvBertForMaskedLM,
     TFConvBertForMultipleChoice,
     TFConvBertForQuestionAnswering,
     TFConvBertForSequenceClassification,
     TFConvBertForTokenClassification,
     TFConvBertModel,
 )






class    a__    :


     def __init__(     self,				_UpperCAmelCase,				_UpperCAmelCase=13,				_UpperCAmelCase=7,				_UpperCAmelCase=True,				_UpperCAmelCase=True,				_UpperCAmelCase=True,				_UpperCAmelCase=True,				_UpperCAmelCase=99,				_UpperCAmelCase=32,				_UpperCAmelCase=2,				_UpperCAmelCase=4,				_UpperCAmelCase=37,				_UpperCAmelCase="gelu",				_UpperCAmelCase=0.1,				_UpperCAmelCase=0.1,				_UpperCAmelCase=512,				_UpperCAmelCase=16,				_UpperCAmelCase=2,				_UpperCAmelCase=0.02,				_UpperCAmelCase=3,				_UpperCAmelCase=4,				_UpperCAmelCase=None,				):
          '''simple docstring'''



          lowercase__				 =							parent
          lowercase__				 =							13
          lowercase__				 =							7
          lowercase__				 =							True
          lowercase__				 =							True
          lowercase__				 =							True
          lowercase__				 =							True
          lowercase__				 =							99
          lowercase__				 =							384
          lowercase__				 =							2
          lowercase__				 =							4
          lowercase__				 =							37
          lowercase__				 =							"gelu"
          lowercase__				 =							0.1
          lowercase__				 =							0.1
          lowercase__				 =							512
          lowercase__				 =							16
          lowercase__				 =							2
          lowercase__				 =							0.02
          lowercase__				 =							3
          lowercase__				 =							4
          lowercase__				 =							128
          lowercase__				 =							2
          lowercase__				 =							9
          lowercase__				 =							1
          lowercase__				 =							None


     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__				 =							ids_tensor([self.batch_size, self.seq_length],				self.vocab_size  )

          lowercase__				 =							None
          if self.use_input_mask:
               lowercase__				 =							random_attention_mask([self.batch_size, self.seq_length]  )

          lowercase__				 =							None
          if self.use_token_type_ids:
               lowercase__				 =							ids_tensor([self.batch_size, self.seq_length],				self.type_vocab_size  )

          lowercase__				 =							None
          lowercase__				 =							None
          lowercase__				 =							None
          if self.use_labels:
               lowercase__				 =							ids_tensor([self.batch_size],				self.type_sequence_label_size  )
               lowercase__				 =							ids_tensor([self.batch_size, self.seq_length],				self.num_labels  )
               lowercase__				 =							ids_tensor([self.batch_size],				self.num_choices  )

          lowercase__				 =							ConvBertConfig(
              vocab_size=self.vocab_size,				hidden_size=self.hidden_size,				num_hidden_layers=self.num_hidden_layers,				num_attention_heads=self.num_attention_heads,				intermediate_size=self.intermediate_size,				hidden_act=self.hidden_act,				hidden_dropout_prob=self.hidden_dropout_prob,				attention_probs_dropout_prob=self.attention_probs_dropout_prob,				max_position_embeddings=self.max_position_embeddings,				type_vocab_size=self.type_vocab_size,				initializer_range=self.initializer_range,				return_dict=_UpperCAmelCase,				)

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


     def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase  ):
          '''simple docstring'''



          lowercase__				 =							TFConvBertModel(config=_UpperCAmelCase  )
          lowercase__				 =							{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}

          lowercase__				 =							[input_ids, input_mask]
          lowercase__				 =							model(_UpperCAmelCase  )

          lowercase__				 =							model(_UpperCAmelCase  )

          self.parent.assertEqual(result.last_hidden_state.shape,				(self.batch_size, self.seq_length, self.hidden_size)  )


     def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase  ):
          '''simple docstring'''



          lowercase__				 =							TFConvBertForMaskedLM(config=_UpperCAmelCase  )
          lowercase__				 =							{
              "input_ids": input_ids,
              "attention_mask": input_mask,
              "token_type_ids": token_type_ids,
          }
          lowercase__				 =							model(_UpperCAmelCase  )
          self.parent.assertEqual(result.logits.shape,				(self.batch_size, self.seq_length, self.vocab_size)  )


     def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase  ):
          '''simple docstring'''



          lowercase__				 =							self.num_labels
          lowercase__				 =							TFConvBertForSequenceClassification(config=_UpperCAmelCase  )
          lowercase__				 =							{
              "input_ids": input_ids,
              "attention_mask": input_mask,
              "token_type_ids": token_type_ids,
          }

          lowercase__				 =							model(_UpperCAmelCase  )
          self.parent.assertEqual(result.logits.shape,				(self.batch_size, self.num_labels)  )


     def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase  ):
          '''simple docstring'''



          lowercase__				 =							self.num_choices
          lowercase__				 =							TFConvBertForMultipleChoice(config=_UpperCAmelCase  )
          lowercase__				 =							tf.tile(tf.expand_dims(_UpperCAmelCase,				1  ),				(1, self.num_choices, 1)  )
          lowercase__				 =							tf.tile(tf.expand_dims(_UpperCAmelCase,				1  ),				(1, self.num_choices, 1)  )
          lowercase__				 =							tf.tile(tf.expand_dims(_UpperCAmelCase,				1  ),				(1, self.num_choices, 1)  )
          lowercase__				 =							{
              "input_ids": multiple_choice_inputs_ids,
              "attention_mask": multiple_choice_input_mask,
              "token_type_ids": multiple_choice_token_type_ids,
          }
          lowercase__				 =							model(_UpperCAmelCase  )
          self.parent.assertEqual(result.logits.shape,				(self.batch_size, self.num_choices)  )


     def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase  ):
          '''simple docstring'''



          lowercase__				 =							self.num_labels
          lowercase__				 =							TFConvBertForTokenClassification(config=_UpperCAmelCase  )
          lowercase__				 =							{
              "input_ids": input_ids,
              "attention_mask": input_mask,
              "token_type_ids": token_type_ids,
          }
          lowercase__				 =							model(_UpperCAmelCase  )
          self.parent.assertEqual(result.logits.shape,				(self.batch_size, self.seq_length, self.num_labels)  )


     def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase,				_UpperCAmelCase  ):
          '''simple docstring'''



          lowercase__				 =							TFConvBertForQuestionAnswering(config=_UpperCAmelCase  )
          lowercase__				 =							{
              "input_ids": input_ids,
              "attention_mask": input_mask,
              "token_type_ids": token_type_ids,
          }

          lowercase__				 =							model(_UpperCAmelCase  )
          self.parent.assertEqual(result.start_logits.shape,				(self.batch_size, self.seq_length)  )
          self.parent.assertEqual(result.end_logits.shape,				(self.batch_size, self.seq_length)  )


     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__				 =							self.prepare_config_and_inputs()
          (
              (
              lowercase__		
          ) ,						(
              lowercase__		
          ) ,						(
              lowercase__		
          ) ,						(
              lowercase__		
          ) ,						(
              lowercase__		
          ) ,						(
              lowercase__		
          ) ,						(
              lowercase__		
          ) ,								
          )				 =							config_and_inputs
          lowercase__				 =							{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
          return config, inputs_dict






@require_tf
class    a__    (   __UpperCAmelCase					,					__UpperCAmelCase					,					unittest.TestCase	):
     snake_case_							=      (
         (
             TFConvBertModel,
             TFConvBertForMaskedLM,
             TFConvBertForQuestionAnswering,
             TFConvBertForSequenceClassification,
             TFConvBertForTokenClassification,
             TFConvBertForMultipleChoice,
         )
         if is_tf_available()
         else ()
     )
     snake_case_							=      (
         {
             "feature-extraction": TFConvBertModel,
             "fill-mask": TFConvBertForMaskedLM,
             "question-answering": TFConvBertForQuestionAnswering,
             "text-classification": TFConvBertForSequenceClassification,
             "token-classification": TFConvBertForTokenClassification,
             "zero-shot": TFConvBertForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     snake_case_							=      False
     snake_case_							=      False
     snake_case_							=      False


     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__				 =							TFConvBertModelTester(self  )
          lowercase__				 =							ConfigTester(self,				config_class=_UpperCAmelCase,				hidden_size=37  )


     def 	snake_case__	(     self  ):
          '''simple docstring'''



          self.config_tester.run_common_tests()


     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__				 =							self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*_UpperCAmelCase  )


     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__				 =							self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase  )


     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__				 =							self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase  )


     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__				 =							self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase  )


     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__				 =							self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase  )


     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__				 =							self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase  )


     @slow
     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__ ,						lowercase__				 =							self.model_tester.prepare_config_and_inputs_for_common()
          lowercase__				 =							True
          lowercase__				 =							True

          if hasattr(_UpperCAmelCase,				"use_cache"  ):
               lowercase__				 =							True

          lowercase__				 =							getattr(self.model_tester,				"encoder_seq_length",				self.model_tester.seq_length  )
          lowercase__				 =							getattr(self.model_tester,				"key_length",				_UpperCAmelCase  )

          for model_class in self.all_model_classes:
               lowercase__				 =							self._prepare_for_class(_UpperCAmelCase,				_UpperCAmelCase  )
               lowercase__				 =							model_class(_UpperCAmelCase  )
               lowercase__				 =							len(model(_UpperCAmelCase  )  )

               with tempfile.TemporaryDirectory() as tmpdirname:
                    model.save_pretrained(_UpperCAmelCase,				saved_model=_UpperCAmelCase  )
                    lowercase__				 =							os.path.join(_UpperCAmelCase,				"saved_model",				"1"  )
                    lowercase__				 =							tf.keras.models.load_model(_UpperCAmelCase  )
                    lowercase__				 =							model(_UpperCAmelCase  )

                    if self.is_encoder_decoder:
                         lowercase__				 =							outputs["encoder_hidden_states"]
                         lowercase__				 =							outputs["encoder_attentions"]
                    else:
                         lowercase__				 =							outputs["hidden_states"]
                         lowercase__				 =							outputs["attentions"]

                    self.assertEqual(len(_UpperCAmelCase  ),				_UpperCAmelCase  )

                    lowercase__				 =							getattr(
                        self.model_tester,				"expected_num_hidden_layers",				self.model_tester.num_hidden_layers + 1  )

                    self.assertEqual(len(_UpperCAmelCase  ),				_UpperCAmelCase  )
                    self.assertListEqual(
                        list(output_hidden_states[0].shape[-2:]  ),				[self.model_tester.seq_length, self.model_tester.hidden_size],				)

                    self.assertEqual(len(_UpperCAmelCase  ),				self.model_tester.num_hidden_layers  )
                    self.assertListEqual(
                        list(output_attentions[0].shape[-3:]  ),				[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length],				)


     @slow
     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__				 =							TFConvBertModel.from_pretrained("YituTech/conv-bert-base"  )
          self.assertIsNotNone(_UpperCAmelCase  )


     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__ ,						lowercase__				 =							self.model_tester.prepare_config_and_inputs_for_common()
          lowercase__				 =							True
          lowercase__				 =							getattr(self.model_tester,				"decoder_seq_length",				self.model_tester.seq_length  )
          lowercase__				 =							getattr(self.model_tester,				"encoder_seq_length",				self.model_tester.seq_length  )
          lowercase__				 =							getattr(self.model_tester,				"key_length",				_UpperCAmelCase  )
          lowercase__				 =							getattr(self.model_tester,				"key_length",				_UpperCAmelCase  )

          def check_decoder_attentions_output(_UpperCAmelCase  ):
               lowercase__				 =							len(_UpperCAmelCase  )
               self.assertEqual(out_len % 2,				0  )
               lowercase__				 =							outputs.decoder_attentions
               self.assertEqual(len(_UpperCAmelCase  ),				self.model_tester.num_hidden_layers  )
               self.assertListEqual(
                   list(decoder_attentions[0].shape[-3:]  ),				[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length],				)

          def check_encoder_attentions_output(_UpperCAmelCase  ):
               lowercase__				 =							[
                   t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
               ]
               self.assertEqual(len(_UpperCAmelCase  ),				self.model_tester.num_hidden_layers  )
               self.assertListEqual(
                   list(attentions[0].shape[-3:]  ),				[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length],				)

          for model_class in self.all_model_classes:
               lowercase__				 =							True
               lowercase__				 =							False
               lowercase__				 =							model_class(_UpperCAmelCase  )
               lowercase__				 =							model(self._prepare_for_class(_UpperCAmelCase,				_UpperCAmelCase  )  )
               lowercase__				 =							len(_UpperCAmelCase  )
               self.assertEqual(config.output_hidden_states,				_UpperCAmelCase  )
               check_encoder_attentions_output(_UpperCAmelCase  )

               if self.is_encoder_decoder:
                    lowercase__				 =							model_class(_UpperCAmelCase  )
                    lowercase__				 =							model(self._prepare_for_class(_UpperCAmelCase,				_UpperCAmelCase  )  )
                    self.assertEqual(config.output_hidden_states,				_UpperCAmelCase  )
                    check_decoder_attentions_output(_UpperCAmelCase  )

               # Check that output attentions can also be changed via the config
               del inputs_dict["output_attentions"]
               lowercase__				 =							True
               lowercase__				 =							model_class(_UpperCAmelCase  )
               lowercase__				 =							model(self._prepare_for_class(_UpperCAmelCase,				_UpperCAmelCase  )  )
               self.assertEqual(config.output_hidden_states,				_UpperCAmelCase  )
               check_encoder_attentions_output(_UpperCAmelCase  )

               # Check attention is always last and order is fine
               lowercase__				 =							True
               lowercase__				 =							True
               lowercase__				 =							model_class(_UpperCAmelCase  )
               lowercase__				 =							model(self._prepare_for_class(_UpperCAmelCase,				_UpperCAmelCase  )  )

               self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1),				len(_UpperCAmelCase  )  )
               self.assertEqual(model.config.output_hidden_states,				_UpperCAmelCase  )
               check_encoder_attentions_output(_UpperCAmelCase  )






@require_tf
class    a__    (   unittest.TestCase	):


     @slow
     def 	snake_case__	(     self  ):
          '''simple docstring'''



          lowercase__				 =							TFConvBertModel.from_pretrained("YituTech/conv-bert-base"  )
          lowercase__				 =							tf.constant([[0, 1, 2, 3, 4, 5]]  )
          lowercase__				 =							model(_UpperCAmelCase  )[0]

          lowercase__				 =							[1, 6, 768]
          self.assertEqual(output.shape,				_UpperCAmelCase  )

          lowercase__				 =							tf.constant(
              [
                  [
                      [-0.03_475_493, -0.4_686_034, -0.30_638_832],
                      [0.22_637_248, -0.26_988_646, -0.7_423_424],
                      [0.10_324_868, -0.45_013_508, -0.58_280_784],
                  ]
              ]  )
          tf.debugging.assert_near(output[:, :3, :3],				_UpperCAmelCase,				atol=1E-4  )





"""simple docstring"""


from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase_:					List[Any]     =					logging.get_logger(__name__)

lowerCAmelCase_:					int     =					{
    "microsoft/markuplm-base": "https://huggingface.co/microsoft/markuplm-base/resolve/main/config.json",
    "microsoft/markuplm-large": "https://huggingface.co/microsoft/markuplm-large/resolve/main/config.json",
}






class    a__    (   _a	):
   snake_case_							=      "markuplm"


   def __init__(     self,				_UpperCAmelCase=3_0522,				_UpperCAmelCase=768,				_UpperCAmelCase=12,				_UpperCAmelCase=12,				_UpperCAmelCase=3072,				_UpperCAmelCase="gelu",				_UpperCAmelCase=0.1,				_UpperCAmelCase=0.1,				_UpperCAmelCase=512,				_UpperCAmelCase=2,				_UpperCAmelCase=0.02,				_UpperCAmelCase=1E-12,				_UpperCAmelCase=0,				_UpperCAmelCase=0,				_UpperCAmelCase=2,				_UpperCAmelCase=256,				_UpperCAmelCase=1024,				_UpperCAmelCase=216,				_UpperCAmelCase=1001,				_UpperCAmelCase=32,				_UpperCAmelCase=50,				_UpperCAmelCase="absolute",				_UpperCAmelCase=True,				_UpperCAmelCase=None,				**_UpperCAmelCase,				):
        '''simple docstring'''



        super().__init__(
            pad_token_id=_UpperCAmelCase,				bos_token_id=_UpperCAmelCase,				eos_token_id=_UpperCAmelCase,				**_UpperCAmelCase,				)
        lowercase__				 =							vocab_size
        lowercase__				 =							hidden_size
        lowercase__				 =							num_hidden_layers
        lowercase__				 =							num_attention_heads
        lowercase__				 =							hidden_act
        lowercase__				 =							intermediate_size
        lowercase__				 =							hidden_dropout_prob
        lowercase__				 =							attention_probs_dropout_prob
        lowercase__				 =							max_position_embeddings
        lowercase__				 =							type_vocab_size
        lowercase__				 =							initializer_range
        lowercase__				 =							layer_norm_eps
        lowercase__				 =							position_embedding_type
        lowercase__				 =							use_cache
        lowercase__				 =							classifier_dropout
        # additional properties
        lowercase__				 =							max_depth
        lowercase__				 =							max_xpath_tag_unit_embeddings
        lowercase__				 =							max_xpath_subs_unit_embeddings
        lowercase__				 =							tag_pad_id
        lowercase__				 =							subs_pad_id
        lowercase__				 =							xpath_unit_hidden_size




