



import inspect
import unittest

import numpy as np

from tests.test_modeling_common import floats_tensor
from transformers import DetrConfig, MaskFormerConfig, SwinConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import MaskFormerForInstanceSegmentation, MaskFormerModel

    if is_vision_available():
        from transformers import MaskFormerImageProcessor

if is_vision_available():
    from PIL import Image





class 		UpperCAmelCase						:





   def __init__(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase=2       ,     _lowerCAmelCase=True       ,     _lowerCAmelCase=False       ,     _lowerCAmelCase=10       ,     _lowerCAmelCase=3       ,     _lowerCAmelCase=32 * 4       ,     _lowerCAmelCase=32 * 6       ,     _lowerCAmelCase=4       ,     _lowerCAmelCase=32       ,     ):
         _lowerCAmelCase 			=  parent
         _lowerCAmelCase 			=  batch_size
         _lowerCAmelCase 			=  is_training
         _lowerCAmelCase 			=  use_auxiliary_loss
         _lowerCAmelCase 			=  num_queries
         _lowerCAmelCase 			=  num_channels
         _lowerCAmelCase 			=  min_size
         _lowerCAmelCase 			=  max_size
         _lowerCAmelCase 			=  num_labels
         _lowerCAmelCase 			=  mask_feature_size





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size]			).to(
             _lowerCAmelCase			)

         _lowerCAmelCase 			=  torch.ones([self.batch_size, self.min_size, self.max_size]       ,     device=_lowerCAmelCase			)

         _lowerCAmelCase 			=  (
             torch.rand([self.batch_size, self.num_labels, self.min_size, self.max_size]       ,     device=_lowerCAmelCase			) > 0.5
         ).float()
         _lowerCAmelCase 			=  (torch.rand((self.batch_size, self.num_labels)       ,     device=_lowerCAmelCase			) > 0.5).long()

         _lowerCAmelCase 			=  self.get_config()
         return config, pixel_values, pixel_mask, mask_labels, class_labels





   def 	__lowerCAmelCase							(	self			):
         return MaskFormerConfig.from_backbone_and_decoder_configs(
             backbone_config=SwinConfig(
                 depths=[1, 1, 1, 1]       ,     )       ,     decoder_config=DetrConfig(
                 decoder_ffn_dim=128       ,     num_queries=self.num_queries       ,     decoder_attention_heads=2       ,     d_model=self.mask_feature_size       ,     )       ,     mask_feature_size=self.mask_feature_size       ,     fpn_feature_size=self.mask_feature_size       ,     num_channels=self.num_channels       ,     num_labels=self.num_labels       ,     )





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase 			=  self.prepare_config_and_inputs()
         _lowerCAmelCase 			=  {'''pixel_values''': pixel_values, '''pixel_mask''': pixel_mask}
         return config, inputs_dict





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  output.encoder_hidden_states
         _lowerCAmelCase 			=  output.pixel_decoder_hidden_states
         _lowerCAmelCase 			=  output.transformer_decoder_hidden_states

         self.parent.assertTrue(len(_lowerCAmelCase			)       ,     len(config.backbone_config.depths			)			)
         self.parent.assertTrue(len(_lowerCAmelCase			)       ,     len(config.backbone_config.depths			)			)
         self.parent.assertTrue(len(_lowerCAmelCase			)       ,     config.decoder_config.decoder_layers			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase=False			):
         with torch.no_grad():
               _lowerCAmelCase 			=  MaskFormerModel(config=_lowerCAmelCase			)
               model.to(_lowerCAmelCase			)
               model.eval()

               _lowerCAmelCase 			=  model(pixel_values=_lowerCAmelCase       ,     pixel_mask=_lowerCAmelCase			)
               _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     output_hidden_states=_lowerCAmelCase			)
         # the correct shape of output.transformer_decoder_hidden_states ensure the correcteness of the
         # encoder and pixel decoder
         self.parent.assertEqual(
             output.transformer_decoder_last_hidden_state.shape       ,     (self.batch_size, self.num_queries, self.mask_feature_size)       ,     )
         # let's ensure the other two hidden state exists
         self.parent.assertTrue(output.pixel_decoder_last_hidden_state is not None			)
         self.parent.assertTrue(output.encoder_last_hidden_state is not None			)

         if output_hidden_states:
               self.check_output_hidden_state(_lowerCAmelCase       ,     _lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  MaskFormerForInstanceSegmentation(config=_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()

         def comm_check_on_output(_lowerCAmelCase			):
               # let's still check that all the required stuff is there
               self.parent.assertTrue(result.transformer_decoder_last_hidden_state is not None			)
               self.parent.assertTrue(result.pixel_decoder_last_hidden_state is not None			)
               self.parent.assertTrue(result.encoder_last_hidden_state is not None			)
               # okay, now we need to check the logits shape
               # due to the encoder compression, masks have a //4 spatial size
               self.parent.assertEqual(
                   result.masks_queries_logits.shape       ,     (self.batch_size, self.num_queries, self.min_size // 4, self.max_size // 4)       ,     )
               # + 1 for null class
               self.parent.assertEqual(
                   result.class_queries_logits.shape       ,     (self.batch_size, self.num_queries, self.num_labels + 1)			)

         with torch.no_grad():
               _lowerCAmelCase 			=  model(pixel_values=_lowerCAmelCase       ,     pixel_mask=_lowerCAmelCase			)
               _lowerCAmelCase 			=  model(_lowerCAmelCase			)

               comm_check_on_output(_lowerCAmelCase			)

               _lowerCAmelCase 			=  model(
                   pixel_values=_lowerCAmelCase       ,     pixel_mask=_lowerCAmelCase       ,     mask_labels=_lowerCAmelCase       ,     class_labels=_lowerCAmelCase			)

         comm_check_on_output(_lowerCAmelCase			)

         self.parent.assertTrue(result.loss is not None			)
         self.parent.assertEqual(result.loss.shape       ,     torch.Size([1]			)			)






@require_torch
class 		UpperCAmelCase						(						snake_case_       ,snake_case_       ,unittest.TestCase				):
   SCREAMING_SNAKE_CASE__							=     (MaskFormerModel, MaskFormerForInstanceSegmentation) if is_torch_available() else ()
   SCREAMING_SNAKE_CASE__							=     (
       {'''feature-extraction''': MaskFormerModel, '''image-segmentation''': MaskFormerForInstanceSegmentation}
       if is_torch_available()
       else {}
   )

   SCREAMING_SNAKE_CASE__							=     False
   SCREAMING_SNAKE_CASE__							=     False
   SCREAMING_SNAKE_CASE__							=     False
   SCREAMING_SNAKE_CASE__							=     False





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  MaskFormerModelTester(self			)
         _lowerCAmelCase 			=  ConfigTester(self       ,     config_class=_lowerCAmelCase       ,     has_text_modality=_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         self.config_tester.run_common_tests()





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase			,   _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs_for_common()
         self.model_tester.create_and_check_maskformer_model(_lowerCAmelCase       ,     **_lowerCAmelCase       ,     output_hidden_states=_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_maskformer_instance_segmentation_head_model(*_lowerCAmelCase			)





   @unittest.skip(reason='''MaskFormer does not use inputs_embeds'''			)
   def 	__lowerCAmelCase							(	self			):
         pass





   @unittest.skip(reason='''MaskFormer does not have a get_input_embeddings method'''			)
   def 	__lowerCAmelCase							(	self			):
         pass





   @unittest.skip(reason='''MaskFormer is not a generative model'''			)
   def 	__lowerCAmelCase							(	self			):
         pass





   @unittest.skip(reason='''MaskFormer does not use token embeddings'''			)
   def 	__lowerCAmelCase							(	self			):
         pass





   @require_torch_multi_gpu
   @unittest.skip(
       reason='''MaskFormer has some layers using `add_module` which doesn\'t work well with `nn.DataParallel`'''			)
   def 	__lowerCAmelCase							(	self			):
         pass





   @unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''			)
   def 	__lowerCAmelCase							(	self			):
         pass





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase			,   _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
               _lowerCAmelCase 			=  model_class(_lowerCAmelCase			)
               _lowerCAmelCase 			=  inspect.signature(model.forward			)
               # signature.parameters is an OrderedDict => so arg_names order is deterministic
               _lowerCAmelCase 			=  [*signature.parameters.keys()]

               _lowerCAmelCase 			=  ['''pixel_values''']
               self.assertListEqual(arg_names[:1]       ,     _lowerCAmelCase			)





   @slow
   def 	__lowerCAmelCase							(	self			):
         for model_name in ["facebook/maskformer-swin-small-coco"]:
               _lowerCAmelCase 			=  MaskFormerModel.from_pretrained(_lowerCAmelCase			)
               self.assertIsNotNone(_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  (self.model_tester.min_size,) * 2
         _lowerCAmelCase 			=  {
             '''pixel_values''': torch.randn((2, 3, *size)       ,     device=_lowerCAmelCase			),
             '''mask_labels''': torch.randn((2, 10, *size)       ,     device=_lowerCAmelCase			),
             '''class_labels''': torch.zeros(2       ,     10       ,     device=_lowerCAmelCase			).long(),
         }

         _lowerCAmelCase 			=  MaskFormerForInstanceSegmentation(MaskFormerConfig()			).to(_lowerCAmelCase			)
         _lowerCAmelCase 			=  model(**_lowerCAmelCase			)
         self.assertTrue(outputs.loss is not None			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase			,   _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs_for_common()
         self.model_tester.create_and_check_maskformer_model(_lowerCAmelCase       ,     **_lowerCAmelCase       ,     output_hidden_states=_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase			,   _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
               _lowerCAmelCase 			=  model_class(_lowerCAmelCase			).to(_lowerCAmelCase			)
               _lowerCAmelCase 			=  model(**_lowerCAmelCase       ,     output_attentions=_lowerCAmelCase			)
               self.assertTrue(outputs.attentions is not None			)





   def 	__lowerCAmelCase							(	self			):
         if not self.model_tester.is_training:
               return
         # only MaskFormerForInstanceSegmentation has the loss
         _lowerCAmelCase 			=  self.all_model_classes[1]
         _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()

         _lowerCAmelCase 			=  model_class(_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.train()

         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     mask_labels=_lowerCAmelCase       ,     class_labels=_lowerCAmelCase			).loss
         loss.backward()





   def 	__lowerCAmelCase							(	self			):
         # only MaskFormerForInstanceSegmentation has the loss
         _lowerCAmelCase 			=  self.all_model_classes[1]
         _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         _lowerCAmelCase 			=  True
         _lowerCAmelCase 			=  True

         _lowerCAmelCase 			=  model_class(_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.train()

         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     mask_labels=_lowerCAmelCase       ,     class_labels=_lowerCAmelCase			)

         _lowerCAmelCase 			=  outputs.encoder_hidden_states[0]
         encoder_hidden_states.retain_grad()

         _lowerCAmelCase 			=  outputs.pixel_decoder_hidden_states[0]
         pixel_decoder_hidden_states.retain_grad()
         # we requires_grad=True in inputs_embeds (line 2152), the original implementation don't
         _lowerCAmelCase 			=  outputs.transformer_decoder_hidden_states[0]
         transformer_decoder_hidden_states.retain_grad()

         _lowerCAmelCase 			=  outputs.attentions[0]
         attentions.retain_grad()

         outputs.loss.backward(retain_graph=_lowerCAmelCase			)

         self.assertIsNotNone(encoder_hidden_states.grad			)
         self.assertIsNotNone(pixel_decoder_hidden_states.grad			)
         self.assertIsNotNone(transformer_decoder_hidden_states.grad			)
         self.assertIsNotNone(attentions.grad			)






UpperCAmelCase_									=							1E-4
def 							UpperCAmelCase__		(				)->Union[str, Any]:
      _lowerCAmelCase 			=  Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''				)
      return image





@require_vision
@slow
class 		UpperCAmelCase						(						unittest.TestCase				):





   @cached_property
   def 	__lowerCAmelCase							(	self			):
         return (
             MaskFormerImageProcessor.from_pretrained('''facebook/maskformer-swin-small-coco'''			)
             if is_vision_available()
             else None
         )





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  MaskFormerModel.from_pretrained('''facebook/maskformer-swin-small-coco'''			).to(_lowerCAmelCase			)
         _lowerCAmelCase 			=  self.default_image_processor
         _lowerCAmelCase 			=  prepare_img()
         _lowerCAmelCase 			=  image_processor(_lowerCAmelCase       ,     return_tensors='''pt'''			).to(_lowerCAmelCase			)
         _lowerCAmelCase 			=  inputs['''pixel_values'''].shape
         # check size is divisible by 32
         self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0			)
         # check size
         self.assertEqual(_lowerCAmelCase       ,     (1, 3, 800, 1_088)			)

         with torch.no_grad():
               _lowerCAmelCase 			=  model(**_lowerCAmelCase			)

         _lowerCAmelCase 			=  torch.tensor(
             [[-0.0_482, 0.9_228, 0.4_951], [-0.2_547, 0.8_017, 0.8_527], [-0.0_069, 0.3_385, -0.0_089]]			).to(_lowerCAmelCase			)
         self.assertTrue(
             torch.allclose(
                 outputs.encoder_last_hidden_state[0, 0, :3, :3]       ,     _lowerCAmelCase       ,     atol=_lowerCAmelCase			)			)

         _lowerCAmelCase 			=  torch.tensor(
             [[-0.8_422, -0.8_434, -0.9_718], [-1.0_144, -0.5_565, -0.4_195], [-1.0_038, -0.4_484, -0.1_961]]			).to(_lowerCAmelCase			)
         self.assertTrue(
             torch.allclose(
                 outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3]       ,     _lowerCAmelCase       ,     atol=_lowerCAmelCase			)			)

         _lowerCAmelCase 			=  torch.tensor(
             [[0.2_852, -0.0_159, 0.9_735], [0.6_254, 0.1_858, 0.8_529], [-0.0_680, -0.4_116, 1.8_413]]			).to(_lowerCAmelCase			)
         self.assertTrue(
             torch.allclose(
                 outputs.transformer_decoder_last_hidden_state[0, :3, :3]       ,     _lowerCAmelCase       ,     atol=_lowerCAmelCase			)			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  (
             MaskFormerForInstanceSegmentation.from_pretrained('''facebook/maskformer-swin-small-coco'''			)
             .to(_lowerCAmelCase			)
             .eval()
         )
         _lowerCAmelCase 			=  self.default_image_processor
         _lowerCAmelCase 			=  prepare_img()
         _lowerCAmelCase 			=  image_processor(_lowerCAmelCase       ,     return_tensors='''pt'''			).to(_lowerCAmelCase			)
         _lowerCAmelCase 			=  inputs['''pixel_values'''].shape
         # check size is divisible by 32
         self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0			)
         # check size
         self.assertEqual(_lowerCAmelCase       ,     (1, 3, 800, 1_088)			)

         with torch.no_grad():
               _lowerCAmelCase 			=  model(**_lowerCAmelCase			)
         # masks_queries_logits
         _lowerCAmelCase 			=  outputs.masks_queries_logits
         self.assertEqual(
             masks_queries_logits.shape       ,     (1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)       ,     )
         _lowerCAmelCase 			=  [
             [-1.3_737_124, -1.7_724_937, -1.9_364_233],
             [-1.5_977_281, -1.9_867_939, -2.1_523_695],
             [-1.5_795_398, -1.9_269_832, -2.093_942],
         ]
         _lowerCAmelCase 			=  torch.tensor(_lowerCAmelCase			).to(_lowerCAmelCase			)
         self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]       ,     _lowerCAmelCase       ,     atol=_lowerCAmelCase			)			)
         # class_queries_logits
         _lowerCAmelCase 			=  outputs.class_queries_logits
         self.assertEqual(
             class_queries_logits.shape       ,     (1, model.config.decoder_config.num_queries, model.config.num_labels + 1)			)
         _lowerCAmelCase 			=  torch.tensor(
             [
                 [1.6512E00, -5.2572E00, -3.3519E00],
                 [3.6169E-02, -5.9025E00, -2.9313E00],
                 [1.0766E-04, -7.7630E00, -5.1263E00],
             ]			).to(_lowerCAmelCase			)
         self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]       ,     _lowerCAmelCase       ,     atol=_lowerCAmelCase			)			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  (
             MaskFormerForInstanceSegmentation.from_pretrained('''facebook/maskformer-resnet101-coco-stuff'''			)
             .to(_lowerCAmelCase			)
             .eval()
         )
         _lowerCAmelCase 			=  self.default_image_processor
         _lowerCAmelCase 			=  prepare_img()
         _lowerCAmelCase 			=  image_processor(_lowerCAmelCase       ,     return_tensors='''pt'''			).to(_lowerCAmelCase			)
         _lowerCAmelCase 			=  inputs['''pixel_values'''].shape
         # check size is divisible by 32
         self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0			)
         # check size
         self.assertEqual(_lowerCAmelCase       ,     (1, 3, 800, 1_088)			)

         with torch.no_grad():
               _lowerCAmelCase 			=  model(**_lowerCAmelCase			)
         # masks_queries_logits
         _lowerCAmelCase 			=  outputs.masks_queries_logits
         self.assertEqual(
             masks_queries_logits.shape       ,     (1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)       ,     )
         _lowerCAmelCase 			=  [[-0.9_046, -2.6_366, -4.6_062], [-3.4_179, -5.7_890, -8.8_057], [-4.9_179, -7.6_560, -10.7_711]]
         _lowerCAmelCase 			=  torch.tensor(_lowerCAmelCase			).to(_lowerCAmelCase			)
         self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]       ,     _lowerCAmelCase       ,     atol=_lowerCAmelCase			)			)
         # class_queries_logits
         _lowerCAmelCase 			=  outputs.class_queries_logits
         self.assertEqual(
             class_queries_logits.shape       ,     (1, model.config.decoder_config.num_queries, model.config.num_labels + 1)			)
         _lowerCAmelCase 			=  torch.tensor(
             [[4.7_188, -3.2_585, -2.8_857], [6.6_871, -2.9_181, -1.2_487], [7.2_449, -2.2_764, -2.1_874]]			).to(_lowerCAmelCase			)
         self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]       ,     _lowerCAmelCase       ,     atol=_lowerCAmelCase			)			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  (
             MaskFormerForInstanceSegmentation.from_pretrained('''facebook/maskformer-swin-small-coco'''			)
             .to(_lowerCAmelCase			)
             .eval()
         )
         _lowerCAmelCase 			=  self.default_image_processor

         _lowerCAmelCase 			=  image_processor(
             [np.zeros((3, 800, 1_333)			), np.zeros((3, 800, 1_333)			)]       ,     segmentation_maps=[np.zeros((384, 384)			).astype(np.floataa			), np.zeros((384, 384)			).astype(np.floataa			)]       ,     return_tensors='''pt'''       ,     )

         _lowerCAmelCase 			=  inputs['''pixel_values'''].to(_lowerCAmelCase			)
         _lowerCAmelCase 			=  [el.to(_lowerCAmelCase			) for el in inputs['''mask_labels''']]
         _lowerCAmelCase 			=  [el.to(_lowerCAmelCase			) for el in inputs['''class_labels''']]

         with torch.no_grad():
               _lowerCAmelCase 			=  model(**_lowerCAmelCase			)

         self.assertTrue(outputs.loss is not None			)




import unittest

from transformers import MraConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_torch_available():
    import torch

    from transformers import (
        MraForMaskedLM,
        MraForMultipleChoice,
        MraForQuestionAnswering,
        MraForSequenceClassification,
        MraForTokenClassification,
        MraModel,
    )
    from transformers.models.mra.modeling_mra import MRA_PRETRAINED_MODEL_ARCHIVE_LIST





class 		UpperCAmelCase						:





   def __init__(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase=2       ,     _lowerCAmelCase=8       ,     _lowerCAmelCase=True       ,     _lowerCAmelCase=True       ,     _lowerCAmelCase=True       ,     _lowerCAmelCase=True       ,     _lowerCAmelCase=99       ,     _lowerCAmelCase=16       ,     _lowerCAmelCase=5       ,     _lowerCAmelCase=2       ,     _lowerCAmelCase=36       ,     _lowerCAmelCase="gelu"       ,     _lowerCAmelCase=0.0       ,     _lowerCAmelCase=0.0       ,     _lowerCAmelCase=512       ,     _lowerCAmelCase=16       ,     _lowerCAmelCase=2       ,     _lowerCAmelCase=0.02       ,     _lowerCAmelCase=3       ,     _lowerCAmelCase=4       ,     _lowerCAmelCase=None       ,     ):
         _lowerCAmelCase 			=  parent
         _lowerCAmelCase 			=  batch_size
         _lowerCAmelCase 			=  seq_length
         _lowerCAmelCase 			=  is_training
         _lowerCAmelCase 			=  use_input_mask
         _lowerCAmelCase 			=  use_token_type_ids
         _lowerCAmelCase 			=  use_labels
         _lowerCAmelCase 			=  vocab_size
         _lowerCAmelCase 			=  hidden_size
         _lowerCAmelCase 			=  num_hidden_layers
         _lowerCAmelCase 			=  num_attention_heads
         _lowerCAmelCase 			=  intermediate_size
         _lowerCAmelCase 			=  hidden_act
         _lowerCAmelCase 			=  hidden_dropout_prob
         _lowerCAmelCase 			=  attention_probs_dropout_prob
         _lowerCAmelCase 			=  max_position_embeddings
         _lowerCAmelCase 			=  type_vocab_size
         _lowerCAmelCase 			=  type_sequence_label_size
         _lowerCAmelCase 			=  initializer_range
         _lowerCAmelCase 			=  num_labels
         _lowerCAmelCase 			=  num_choices
         _lowerCAmelCase 			=  scope





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  ids_tensor([self.batch_size, self.seq_length]       ,     self.vocab_size			)

         _lowerCAmelCase 			=  None
         if self.use_input_mask:
               _lowerCAmelCase 			=  random_attention_mask([self.batch_size, self.seq_length]			)

         _lowerCAmelCase 			=  None
         if self.use_token_type_ids:
               _lowerCAmelCase 			=  ids_tensor([self.batch_size, self.seq_length]       ,     self.type_vocab_size			)

         _lowerCAmelCase 			=  None
         _lowerCAmelCase 			=  None
         _lowerCAmelCase 			=  None
         if self.use_labels:
               _lowerCAmelCase 			=  ids_tensor([self.batch_size]       ,     self.type_sequence_label_size			)
               _lowerCAmelCase 			=  ids_tensor([self.batch_size, self.seq_length]       ,     self.num_labels			)
               _lowerCAmelCase 			=  ids_tensor([self.batch_size]       ,     self.num_choices			)

         _lowerCAmelCase 			=  self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





   def 	__lowerCAmelCase							(	self			):
         return MraConfig(
             vocab_size=self.vocab_size       ,     hidden_size=self.hidden_size       ,     num_hidden_layers=self.num_hidden_layers       ,     num_attention_heads=self.num_attention_heads       ,     intermediate_size=self.intermediate_size       ,     hidden_act=self.hidden_act       ,     hidden_dropout_prob=self.hidden_dropout_prob       ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,     max_position_embeddings=self.max_position_embeddings       ,     type_vocab_size=self.type_vocab_size       ,     is_decoder=_lowerCAmelCase       ,     initializer_range=self.initializer_range       ,     )





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.get_config()
         _lowerCAmelCase 			=  300
         return config





   def 	__lowerCAmelCase							(	self			):
         (
             (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   
         ) 			=  self.prepare_config_and_inputs()

         _lowerCAmelCase 			=  True
         _lowerCAmelCase 			=  floats_tensor([self.batch_size, self.seq_length, self.hidden_size]			)
         _lowerCAmelCase 			=  ids_tensor([self.batch_size, self.seq_length]       ,     vocab_size=2			)

         return (
             config,
             input_ids,
             token_type_ids,
             input_mask,
             sequence_labels,
             token_labels,
             choice_labels,
             encoder_hidden_states,
             encoder_attention_mask,
         )





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  MraModel(config=_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase			)
         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase			)
         _lowerCAmelCase 			=  model(_lowerCAmelCase			)
         self.parent.assertEqual(result.last_hidden_state.shape       ,     (self.batch_size, self.seq_length, self.hidden_size)			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     ):
         _lowerCAmelCase 			=  True
         _lowerCAmelCase 			=  MraModel(_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  model(
             _lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     encoder_hidden_states=_lowerCAmelCase       ,     encoder_attention_mask=_lowerCAmelCase       ,     )
         _lowerCAmelCase 			=  model(
             _lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     encoder_hidden_states=_lowerCAmelCase       ,     )
         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase			)
         self.parent.assertEqual(result.last_hidden_state.shape       ,     (self.batch_size, self.seq_length, self.hidden_size)			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  MraForMaskedLM(config=_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     labels=_lowerCAmelCase			)
         self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.seq_length, self.vocab_size)			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  MraForQuestionAnswering(config=_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  model(
             _lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     start_positions=_lowerCAmelCase       ,     end_positions=_lowerCAmelCase       ,     )
         self.parent.assertEqual(result.start_logits.shape       ,     (self.batch_size, self.seq_length)			)
         self.parent.assertEqual(result.end_logits.shape       ,     (self.batch_size, self.seq_length)			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  self.num_labels
         _lowerCAmelCase 			=  MraForSequenceClassification(_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     labels=_lowerCAmelCase			)
         self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.num_labels)			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  self.num_labels
         _lowerCAmelCase 			=  MraForTokenClassification(config=_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     labels=_lowerCAmelCase			)
         self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.seq_length, self.num_labels)			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  self.num_choices
         _lowerCAmelCase 			=  MraForMultipleChoice(config=_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  input_ids.unsqueeze(1			).expand(-1       ,     self.num_choices       ,     -1			).contiguous()
         _lowerCAmelCase 			=  token_type_ids.unsqueeze(1			).expand(-1       ,     self.num_choices       ,     -1			).contiguous()
         _lowerCAmelCase 			=  input_mask.unsqueeze(1			).expand(-1       ,     self.num_choices       ,     -1			).contiguous()
         _lowerCAmelCase 			=  model(
             _lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     labels=_lowerCAmelCase       ,     )
         self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.num_choices)			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.prepare_config_and_inputs()
         (
             (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   
         ) 			=  config_and_inputs
         _lowerCAmelCase 			=  {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
         return config, inputs_dict






@require_torch
class 		UpperCAmelCase						(						snake_case_       ,unittest.TestCase				):
   SCREAMING_SNAKE_CASE__							=     (
       (
           MraModel,
           MraForMaskedLM,
           MraForMultipleChoice,
           MraForQuestionAnswering,
           MraForSequenceClassification,
           MraForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   SCREAMING_SNAKE_CASE__							=     False
   SCREAMING_SNAKE_CASE__							=     False
   SCREAMING_SNAKE_CASE__							=     False
   SCREAMING_SNAKE_CASE__							=     False

   SCREAMING_SNAKE_CASE__							=     ()





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  MraModelTester(self			)
         _lowerCAmelCase 			=  ConfigTester(self       ,     config_class=_lowerCAmelCase       ,     hidden_size=37			)





   def 	__lowerCAmelCase							(	self			):
         self.config_tester.run_common_tests()





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
               _lowerCAmelCase 			=  type
               self.model_tester.create_and_check_model(*_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*_lowerCAmelCase			)





   @slow
   def 	__lowerCAmelCase							(	self			):
         for model_name in MRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               _lowerCAmelCase 			=  MraModel.from_pretrained(_lowerCAmelCase			)
               self.assertIsNotNone(_lowerCAmelCase			)





   @unittest.skip(reason='''MRA does not output attentions'''			)
   def 	__lowerCAmelCase							(	self			):
         return






@require_torch
class 		UpperCAmelCase						(						unittest.TestCase				):





   @slow
   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  MraModel.from_pretrained('''uw-madison/mra-base-512-4'''			)
         _lowerCAmelCase 			=  torch.arange(256			).unsqueeze(0			)

         with torch.no_grad():
               _lowerCAmelCase 			=  model(_lowerCAmelCase			)[0]

         _lowerCAmelCase 			=  torch.Size((1, 256, 768)			)
         self.assertEqual(output.shape       ,     _lowerCAmelCase			)

         _lowerCAmelCase 			=  torch.tensor(
             [[[-0.0_140, 0.0_830, -0.0_381], [0.1_546, 0.1_402, 0.0_220], [0.1_162, 0.0_851, 0.0_165]]]			)

         self.assertTrue(torch.allclose(output[:, :3, :3]       ,     _lowerCAmelCase       ,     atol=1E-4			)			)





   @slow
   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  MraForMaskedLM.from_pretrained('''uw-madison/mra-base-512-4'''			)
         _lowerCAmelCase 			=  torch.arange(256			).unsqueeze(0			)

         with torch.no_grad():
               _lowerCAmelCase 			=  model(_lowerCAmelCase			)[0]

         _lowerCAmelCase 			=  50_265

         _lowerCAmelCase 			=  torch.Size((1, 256, vocab_size)			)
         self.assertEqual(output.shape       ,     _lowerCAmelCase			)

         _lowerCAmelCase 			=  torch.tensor(
             [[[9.2_595, -3.6_038, 11.8_819], [9.3_869, -3.2_693, 11.0_956], [11.8_524, -3.4_938, 13.1_210]]]			)

         self.assertTrue(torch.allclose(output[:, :3, :3]       ,     _lowerCAmelCase       ,     atol=1E-4			)			)





   @slow
   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  MraForMaskedLM.from_pretrained('''uw-madison/mra-base-4096-8-d3'''			)
         _lowerCAmelCase 			=  torch.arange(4_096			).unsqueeze(0			)

         with torch.no_grad():
               _lowerCAmelCase 			=  model(_lowerCAmelCase			)[0]

         _lowerCAmelCase 			=  50_265

         _lowerCAmelCase 			=  torch.Size((1, 4_096, vocab_size)			)
         self.assertEqual(output.shape       ,     _lowerCAmelCase			)

         _lowerCAmelCase 			=  torch.tensor(
             [[[5.4_789, -2.3_564, 7.5_064], [7.9_067, -1.3_369, 9.9_668], [9.0_712, -1.8_106, 7.0_380]]]			)

         self.assertTrue(torch.allclose(output[:, :3, :3]       ,     _lowerCAmelCase       ,     atol=1E-4			)			)