



import argparse
import json
import math
import os
import time
import traceback
import zipfile
from collections import Counter

import requests
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	List[Any]     ,				_SCREAMING_SNAKE_CASE   :	str=None				)->Optional[Any]:
      _lowerCAmelCase 			=  None
      if token is not None:
            _lowerCAmelCase 			=  {"""Accept""": """application/vnd.github+json""", """Authorization""": f'''Bearer {token}'''}

      _lowerCAmelCase 			=  f'''https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs?per_page=100'''
      _lowerCAmelCase 			=  requests.get(lowerCamelCase_     ,				headers=lowerCamelCase_				).json()
      _lowerCAmelCase 			=  {}

      try:
            job_links.update({job['''name''']: job['''html_url'''] for job in result['''jobs''']}				)
            _lowerCAmelCase 			=  math.ceil((result['''total_count'''] - 1_0_0) / 1_0_0				)

            for i in range(lowerCamelCase_				):
                  _lowerCAmelCase 			=  requests.get(url + f'''&page={i + 2}'''     ,				headers=lowerCamelCase_				).json()
                  job_links.update({job['''name''']: job['''html_url'''] for job in result['''jobs''']}				)

            return job_links
      except Exception:
            print(f'''Unknown error, could not fetch links:\n{traceback.format_exc()}'''				)

      return {}
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Union[str, Any]     ,				_SCREAMING_SNAKE_CASE   :	Optional[int]=None				)->int:
      _lowerCAmelCase 			=  None
      if token is not None:
            _lowerCAmelCase 			=  {"""Accept""": """application/vnd.github+json""", """Authorization""": f'''Bearer {token}'''}

      _lowerCAmelCase 			=  f'''https://api.github.com/repos/huggingface/transformers/actions/runs/{worflow_run_id}/artifacts?per_page=100'''
      _lowerCAmelCase 			=  requests.get(lowerCamelCase_     ,				headers=lowerCamelCase_				).json()
      _lowerCAmelCase 			=  {}

      try:
            artifacts.update({artifact['''name''']: artifact['''archive_download_url'''] for artifact in result['''artifacts''']}				)
            _lowerCAmelCase 			=  math.ceil((result['''total_count'''] - 1_0_0) / 1_0_0				)

            for i in range(lowerCamelCase_				):
                  _lowerCAmelCase 			=  requests.get(url + f'''&page={i + 2}'''     ,				headers=lowerCamelCase_				).json()
                  artifacts.update({artifact['''name''']: artifact['''archive_download_url'''] for artifact in result['''artifacts''']}				)

            return artifacts
      except Exception:
            print(f'''Unknown error, could not fetch links:\n{traceback.format_exc()}'''				)

      return {}
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	List[Any]     ,				_SCREAMING_SNAKE_CASE   :	Optional[int]     ,				_SCREAMING_SNAKE_CASE   :	Tuple     ,				_SCREAMING_SNAKE_CASE   :	int				)->List[str]:
      _lowerCAmelCase 			=  None
      if token is not None:
            _lowerCAmelCase 			=  {"""Accept""": """application/vnd.github+json""", """Authorization""": f'''Bearer {token}'''}

      _lowerCAmelCase 			=  requests.get(lowerCamelCase_     ,				headers=lowerCamelCase_     ,				allow_redirects=lowerCamelCase_				)
      _lowerCAmelCase 			=  result.headers["""Location"""]
      _lowerCAmelCase 			=  requests.get(lowerCamelCase_     ,				allow_redirects=lowerCamelCase_				)
      _lowerCAmelCase 			=  os.path.join(lowerCamelCase_     ,				f'''{artifact_name}.zip'''				)
      with open(lowerCamelCase_     ,				'''wb'''				) as fp:
            fp.write(response.content				)
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	List[str]     ,				_SCREAMING_SNAKE_CASE   :	Any=None				)->Union[str, Any]:
      _lowerCAmelCase 			=  []
      _lowerCAmelCase 			=  []
      _lowerCAmelCase 			=  None

      with zipfile.ZipFile(lowerCamelCase_				) as z:
            for filename in z.namelist():
                  if not os.path.isdir(lowerCamelCase_				):
                        # read the file
                        if filename in ["failures_line.txt", "summary_short.txt", "job_name.txt"]:
                              with z.open(lowerCamelCase_				) as f:
                                    for line in f:
                                          _lowerCAmelCase 			=  line.decode('''UTF-8'''				).strip()
                                          if filename == "failures_line.txt":
                                                try:
                                                      # `error_line` is the place where `error` occurs
                                                      _lowerCAmelCase 			=  line[: line.index(''': '''				)]
                                                      _lowerCAmelCase 			=  line[line.index(''': '''				) + len(''': '''				) :]
                                                      errors.append([error_line, error]				)
                                                except Exception:
                                                      # skip un-related lines
                                                      pass
                                          elif filename == "summary_short.txt" and line.startswith('''FAILED '''				):
                                                # `test` is the test method that failed
                                                _lowerCAmelCase 			=  line[len('''FAILED '''				) :]
                                                failed_tests.append(lowerCamelCase_				)
                                          elif filename == "job_name.txt":
                                                _lowerCAmelCase 			=  line

      if len(lowerCamelCase_				) != len(lowerCamelCase_				):
            raise ValueError(
                f'''`errors` and `failed_tests` should have the same number of elements. Got {len(lowerCamelCase_				)} for `errors` '''
                f'''and {len(lowerCamelCase_				)} for `failed_tests` instead. The test reports in {artifact_zip_path} have some'''
                ''' problem.'''				)

      _lowerCAmelCase 			=  None
      if job_name and job_links:
            _lowerCAmelCase 			=  job_links.get(lowerCamelCase_     ,				lowerCamelCase_				)

      # A list with elements of the form (line of error, error, failed test)
      _lowerCAmelCase 			=  [x + [y] + [job_link] for x, y in zip(lowerCamelCase_     ,				lowerCamelCase_				)]

      return result
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Dict     ,				_SCREAMING_SNAKE_CASE   :	int=None				)->Optional[int]:
      _lowerCAmelCase 			=  []

      _lowerCAmelCase 			=  [os.path.join(lowerCamelCase_     ,				lowerCamelCase_				) for p in os.listdir(lowerCamelCase_				) if p.endswith('''.zip'''				)]
      for p in paths:
            errors.extend(get_errors_from_single_artifact(lowerCamelCase_     ,				job_links=lowerCamelCase_				)				)

      return errors
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Any     ,				_SCREAMING_SNAKE_CASE   :	List[Any]=None				)->Tuple:
      _lowerCAmelCase 			=  Counter()
      counter.update([x[1] for x in logs]				)
      _lowerCAmelCase 			=  counter.most_common()
      _lowerCAmelCase 			=  {}
      for error, count in counts:
            if error_filter is None or error not in error_filter:
                  _lowerCAmelCase 			=  {"""count""": count, """failed_tests""": [(x[2], x[0]) for x in logs if x[1] == error]}

      _lowerCAmelCase 			=  dict(sorted(r.items()     ,				key=lambda _SCREAMING_SNAKE_CASE				: item[1]["count"]     ,				reverse=lowerCamelCase_				)				)
      return r
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Tuple				)->List[Any]:
      _lowerCAmelCase 			=  test.split('''::'''				)[0]
      if test.startswith('''tests/models/'''				):
            _lowerCAmelCase 			=  test.split('''/'''				)[2]
      else:
            _lowerCAmelCase 			=  None

      return test
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	List[Any]     ,				_SCREAMING_SNAKE_CASE   :	List[str]=None				)->Dict:
      _lowerCAmelCase 			=  [(x[0], x[1], get_model(x[2]				)) for x in logs]
      _lowerCAmelCase 			=  [x for x in logs if x[2] is not None]
      _lowerCAmelCase 			=  {x[2] for x in logs}

      _lowerCAmelCase 			=  {}
      for test in tests:
            _lowerCAmelCase 			=  Counter()
            # count by errors in `test`
            counter.update([x[1] for x in logs if x[2] == test]				)
            _lowerCAmelCase 			=  counter.most_common()
            _lowerCAmelCase 			=  {error: count for error, count in counts if (error_filter is None or error not in error_filter)}
            _lowerCAmelCase 			=  sum(error_counts.values()				)
            if n_errors > 0:
                  _lowerCAmelCase 			=  {"""count""": n_errors, """errors""": error_counts}

      _lowerCAmelCase 			=  dict(sorted(r.items()     ,				key=lambda _SCREAMING_SNAKE_CASE				: item[1]["count"]     ,				reverse=lowerCamelCase_				)				)
      return r
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Optional[int]				)->Any:
      _lowerCAmelCase 			=  """| no. | error | status |"""
      _lowerCAmelCase 			=  """|-:|:-|:-|"""
      _lowerCAmelCase 			=  [header, sep]
      for error in reduced_by_error:
            _lowerCAmelCase 			=  reduced_by_error[error]["""count"""]
            _lowerCAmelCase 			=  f'''| {count} | {error[:1_0_0]} |  |'''
            lines.append(lowerCamelCase_				)

      return "\n".join(lowerCamelCase_				)
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Dict				)->Union[str, Any]:
      _lowerCAmelCase 			=  """| model | no. of errors | major error | count |"""
      _lowerCAmelCase 			=  """|-:|-:|-:|-:|"""
      _lowerCAmelCase 			=  [header, sep]
      for model in reduced_by_model:
            _lowerCAmelCase 			=  reduced_by_model[model]["""count"""]
            _lowerCAmelCase 			=  list(reduced_by_model[model]['''errors'''].items()				)[0]
            _lowerCAmelCase 			=  f'''| {model} | {count} | {error[:6_0]} | {_count} |'''
            lines.append(lowerCamelCase_				)

      return "\n".join(lowerCamelCase_				)


if __name__ == "__main__":
    UpperCAmelCase_									=							argparse.ArgumentParser()
    # Required parameters
    parser.add_argument("--workflow_run_id", type=str, required=True, help="A GitHub Actions workflow run id.")
    parser.add_argument(
        "--output_dir",
        type=str,
        required=True,
        help="Where to store the downloaded artifacts and other result files.",
    )
    parser.add_argument("--token", default=None, type=str, help="A token that has actions:read permission.")
    UpperCAmelCase_									=							parser.parse_args()

    os.makedirs(args.output_dir, exist_ok=True)

    UpperCAmelCase_									=							get_job_links(args.workflow_run_id, token=args.token)
    UpperCAmelCase_									=							{}
    # To deal with `workflow_call` event, where a job name is the combination of the job names in the caller and callee.
    # For example, `PyTorch 1.11 / Model tests (models/albert, single-gpu)`.
    if _job_links:
        for k, v in _job_links.items():
            # This is how GitHub actions combine job names.
            if " / " in k:
                UpperCAmelCase_									=							k.find(" / ")
                UpperCAmelCase_									=							k[index + len(" / ") :]
            UpperCAmelCase_									=							v
    with open(os.path.join(args.output_dir, "job_links.json"), "w", encoding="UTF-8") as fp:
        json.dump(job_links, fp, ensure_ascii=False, indent=4)

    UpperCAmelCase_									=							get_artifacts_links(args.workflow_run_id, token=args.token)
    with open(os.path.join(args.output_dir, "artifacts.json"), "w", encoding="UTF-8") as fp:
        json.dump(artifacts, fp, ensure_ascii=False, indent=4)

    for idx, (name, url) in enumerate(artifacts.items()):
        download_artifact(name, url, args.output_dir, args.token)
        # Be gentle to GitHub
        time.sleep(1)

    UpperCAmelCase_									=							get_all_errors(args.output_dir, job_links=job_links)

    # `e[1]` is the error
    UpperCAmelCase_									=							Counter()
    counter.update([e[1] for e in errors])

    # print the top 30 most common test errors
    UpperCAmelCase_									=							counter.most_common(3_0)
    for item in most_common:
        print(item)

    with open(os.path.join(args.output_dir, "errors.json"), "w", encoding="UTF-8") as fp:
        json.dump(errors, fp, ensure_ascii=False, indent=4)

    UpperCAmelCase_									=							reduce_by_error(errors)
    UpperCAmelCase_									=							reduce_by_model(errors)

    UpperCAmelCase_									=							make_github_table(reduced_by_error)
    UpperCAmelCase_									=							make_github_table_per_model(reduced_by_model)

    with open(os.path.join(args.output_dir, "reduced_by_error.txt"), "w", encoding="UTF-8") as fp:
        fp.write(sa)
    with open(os.path.join(args.output_dir, "reduced_by_model.txt"), "w", encoding="UTF-8") as fp:
        fp.write(sa)




import unittest

from transformers import MraConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_torch_available():
    import torch

    from transformers import (
        MraForMaskedLM,
        MraForMultipleChoice,
        MraForQuestionAnswering,
        MraForSequenceClassification,
        MraForTokenClassification,
        MraModel,
    )
    from transformers.models.mra.modeling_mra import MRA_PRETRAINED_MODEL_ARCHIVE_LIST





class 		UpperCAmelCase						:





   def __init__(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase=2       ,     _lowerCAmelCase=8       ,     _lowerCAmelCase=True       ,     _lowerCAmelCase=True       ,     _lowerCAmelCase=True       ,     _lowerCAmelCase=True       ,     _lowerCAmelCase=99       ,     _lowerCAmelCase=16       ,     _lowerCAmelCase=5       ,     _lowerCAmelCase=2       ,     _lowerCAmelCase=36       ,     _lowerCAmelCase="gelu"       ,     _lowerCAmelCase=0.0       ,     _lowerCAmelCase=0.0       ,     _lowerCAmelCase=512       ,     _lowerCAmelCase=16       ,     _lowerCAmelCase=2       ,     _lowerCAmelCase=0.02       ,     _lowerCAmelCase=3       ,     _lowerCAmelCase=4       ,     _lowerCAmelCase=None       ,     ):
         _lowerCAmelCase 			=  parent
         _lowerCAmelCase 			=  batch_size
         _lowerCAmelCase 			=  seq_length
         _lowerCAmelCase 			=  is_training
         _lowerCAmelCase 			=  use_input_mask
         _lowerCAmelCase 			=  use_token_type_ids
         _lowerCAmelCase 			=  use_labels
         _lowerCAmelCase 			=  vocab_size
         _lowerCAmelCase 			=  hidden_size
         _lowerCAmelCase 			=  num_hidden_layers
         _lowerCAmelCase 			=  num_attention_heads
         _lowerCAmelCase 			=  intermediate_size
         _lowerCAmelCase 			=  hidden_act
         _lowerCAmelCase 			=  hidden_dropout_prob
         _lowerCAmelCase 			=  attention_probs_dropout_prob
         _lowerCAmelCase 			=  max_position_embeddings
         _lowerCAmelCase 			=  type_vocab_size
         _lowerCAmelCase 			=  type_sequence_label_size
         _lowerCAmelCase 			=  initializer_range
         _lowerCAmelCase 			=  num_labels
         _lowerCAmelCase 			=  num_choices
         _lowerCAmelCase 			=  scope





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  ids_tensor([self.batch_size, self.seq_length]       ,     self.vocab_size			)

         _lowerCAmelCase 			=  None
         if self.use_input_mask:
               _lowerCAmelCase 			=  random_attention_mask([self.batch_size, self.seq_length]			)

         _lowerCAmelCase 			=  None
         if self.use_token_type_ids:
               _lowerCAmelCase 			=  ids_tensor([self.batch_size, self.seq_length]       ,     self.type_vocab_size			)

         _lowerCAmelCase 			=  None
         _lowerCAmelCase 			=  None
         _lowerCAmelCase 			=  None
         if self.use_labels:
               _lowerCAmelCase 			=  ids_tensor([self.batch_size]       ,     self.type_sequence_label_size			)
               _lowerCAmelCase 			=  ids_tensor([self.batch_size, self.seq_length]       ,     self.num_labels			)
               _lowerCAmelCase 			=  ids_tensor([self.batch_size]       ,     self.num_choices			)

         _lowerCAmelCase 			=  self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





   def 	__lowerCAmelCase							(	self			):
         return MraConfig(
             vocab_size=self.vocab_size       ,     hidden_size=self.hidden_size       ,     num_hidden_layers=self.num_hidden_layers       ,     num_attention_heads=self.num_attention_heads       ,     intermediate_size=self.intermediate_size       ,     hidden_act=self.hidden_act       ,     hidden_dropout_prob=self.hidden_dropout_prob       ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,     max_position_embeddings=self.max_position_embeddings       ,     type_vocab_size=self.type_vocab_size       ,     is_decoder=_lowerCAmelCase       ,     initializer_range=self.initializer_range       ,     )





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.get_config()
         _lowerCAmelCase 			=  300
         return config





   def 	__lowerCAmelCase							(	self			):
         (
             (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   
         ) 			=  self.prepare_config_and_inputs()

         _lowerCAmelCase 			=  True
         _lowerCAmelCase 			=  floats_tensor([self.batch_size, self.seq_length, self.hidden_size]			)
         _lowerCAmelCase 			=  ids_tensor([self.batch_size, self.seq_length]       ,     vocab_size=2			)

         return (
             config,
             input_ids,
             token_type_ids,
             input_mask,
             sequence_labels,
             token_labels,
             choice_labels,
             encoder_hidden_states,
             encoder_attention_mask,
         )





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  MraModel(config=_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase			)
         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase			)
         _lowerCAmelCase 			=  model(_lowerCAmelCase			)
         self.parent.assertEqual(result.last_hidden_state.shape       ,     (self.batch_size, self.seq_length, self.hidden_size)			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     ):
         _lowerCAmelCase 			=  True
         _lowerCAmelCase 			=  MraModel(_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  model(
             _lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     encoder_hidden_states=_lowerCAmelCase       ,     encoder_attention_mask=_lowerCAmelCase       ,     )
         _lowerCAmelCase 			=  model(
             _lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     encoder_hidden_states=_lowerCAmelCase       ,     )
         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase			)
         self.parent.assertEqual(result.last_hidden_state.shape       ,     (self.batch_size, self.seq_length, self.hidden_size)			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  MraForMaskedLM(config=_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     labels=_lowerCAmelCase			)
         self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.seq_length, self.vocab_size)			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  MraForQuestionAnswering(config=_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  model(
             _lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     start_positions=_lowerCAmelCase       ,     end_positions=_lowerCAmelCase       ,     )
         self.parent.assertEqual(result.start_logits.shape       ,     (self.batch_size, self.seq_length)			)
         self.parent.assertEqual(result.end_logits.shape       ,     (self.batch_size, self.seq_length)			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  self.num_labels
         _lowerCAmelCase 			=  MraForSequenceClassification(_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     labels=_lowerCAmelCase			)
         self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.num_labels)			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  self.num_labels
         _lowerCAmelCase 			=  MraForTokenClassification(config=_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  model(_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     labels=_lowerCAmelCase			)
         self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.seq_length, self.num_labels)			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  self.num_choices
         _lowerCAmelCase 			=  MraForMultipleChoice(config=_lowerCAmelCase			)
         model.to(_lowerCAmelCase			)
         model.eval()
         _lowerCAmelCase 			=  input_ids.unsqueeze(1			).expand(-1       ,     self.num_choices       ,     -1			).contiguous()
         _lowerCAmelCase 			=  token_type_ids.unsqueeze(1			).expand(-1       ,     self.num_choices       ,     -1			).contiguous()
         _lowerCAmelCase 			=  input_mask.unsqueeze(1			).expand(-1       ,     self.num_choices       ,     -1			).contiguous()
         _lowerCAmelCase 			=  model(
             _lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     token_type_ids=_lowerCAmelCase       ,     labels=_lowerCAmelCase       ,     )
         self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.num_choices)			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.prepare_config_and_inputs()
         (
             (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   (
             _lowerCAmelCase
         )			,   
         ) 			=  config_and_inputs
         _lowerCAmelCase 			=  {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
         return config, inputs_dict






@require_torch
class 		UpperCAmelCase						(						snake_case_       ,unittest.TestCase				):
   SCREAMING_SNAKE_CASE__							=     (
       (
           MraModel,
           MraForMaskedLM,
           MraForMultipleChoice,
           MraForQuestionAnswering,
           MraForSequenceClassification,
           MraForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   SCREAMING_SNAKE_CASE__							=     False
   SCREAMING_SNAKE_CASE__							=     False
   SCREAMING_SNAKE_CASE__							=     False
   SCREAMING_SNAKE_CASE__							=     False

   SCREAMING_SNAKE_CASE__							=     ()





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  MraModelTester(self			)
         _lowerCAmelCase 			=  ConfigTester(self       ,     config_class=_lowerCAmelCase       ,     hidden_size=37			)





   def 	__lowerCAmelCase							(	self			):
         self.config_tester.run_common_tests()





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
               _lowerCAmelCase 			=  type
               self.model_tester.create_and_check_model(*_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*_lowerCAmelCase			)





   @slow
   def 	__lowerCAmelCase							(	self			):
         for model_name in MRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               _lowerCAmelCase 			=  MraModel.from_pretrained(_lowerCAmelCase			)
               self.assertIsNotNone(_lowerCAmelCase			)





   @unittest.skip(reason='''MRA does not output attentions'''			)
   def 	__lowerCAmelCase							(	self			):
         return






@require_torch
class 		UpperCAmelCase						(						unittest.TestCase				):





   @slow
   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  MraModel.from_pretrained('''uw-madison/mra-base-512-4'''			)
         _lowerCAmelCase 			=  torch.arange(256			).unsqueeze(0			)

         with torch.no_grad():
               _lowerCAmelCase 			=  model(_lowerCAmelCase			)[0]

         _lowerCAmelCase 			=  torch.Size((1, 256, 768)			)
         self.assertEqual(output.shape       ,     _lowerCAmelCase			)

         _lowerCAmelCase 			=  torch.tensor(
             [[[-0.0_140, 0.0_830, -0.0_381], [0.1_546, 0.1_402, 0.0_220], [0.1_162, 0.0_851, 0.0_165]]]			)

         self.assertTrue(torch.allclose(output[:, :3, :3]       ,     _lowerCAmelCase       ,     atol=1E-4			)			)





   @slow
   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  MraForMaskedLM.from_pretrained('''uw-madison/mra-base-512-4'''			)
         _lowerCAmelCase 			=  torch.arange(256			).unsqueeze(0			)

         with torch.no_grad():
               _lowerCAmelCase 			=  model(_lowerCAmelCase			)[0]

         _lowerCAmelCase 			=  50_265

         _lowerCAmelCase 			=  torch.Size((1, 256, vocab_size)			)
         self.assertEqual(output.shape       ,     _lowerCAmelCase			)

         _lowerCAmelCase 			=  torch.tensor(
             [[[9.2_595, -3.6_038, 11.8_819], [9.3_869, -3.2_693, 11.0_956], [11.8_524, -3.4_938, 13.1_210]]]			)

         self.assertTrue(torch.allclose(output[:, :3, :3]       ,     _lowerCAmelCase       ,     atol=1E-4			)			)





   @slow
   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  MraForMaskedLM.from_pretrained('''uw-madison/mra-base-4096-8-d3'''			)
         _lowerCAmelCase 			=  torch.arange(4_096			).unsqueeze(0			)

         with torch.no_grad():
               _lowerCAmelCase 			=  model(_lowerCAmelCase			)[0]

         _lowerCAmelCase 			=  50_265

         _lowerCAmelCase 			=  torch.Size((1, 4_096, vocab_size)			)
         self.assertEqual(output.shape       ,     _lowerCAmelCase			)

         _lowerCAmelCase 			=  torch.tensor(
             [[[5.4_789, -2.3_564, 7.5_064], [7.9_067, -1.3_369, 9.9_668], [9.0_712, -1.8_106, 7.0_380]]]			)

         self.assertTrue(torch.allclose(output[:, :3, :3]       ,     _lowerCAmelCase       ,     atol=1E-4			)			)