


from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_=None  ,						lowerCAmelCase_=None   )						->Union[str, Any]:
   if attention_mask is None:
      UpperCAmelCase							=     tf.cast(tf.math.not_equal(lowerCAmelCase_  ,						config.pad_token_id   )  ,						tf.inta   )
   return {"input_ids": input_ids, "attention_mask": attention_mask}






@require_tf
class   __lowercase				:
      UpperCamelCase								=       OPTConfig
      UpperCamelCase								=       {}
      UpperCamelCase								=       '''gelu'''





      def __init__(		self						:					Dict					,       __lowerCamelCase						:					Any					,       __lowerCamelCase						:					Union[str, Any]=1_3					,       __lowerCamelCase						:					int=7					,       __lowerCamelCase						:					List[str]=True					,       __lowerCamelCase						:					Optional[Any]=False					,       __lowerCamelCase						:					Any=9_9					,       __lowerCamelCase						:					Optional[int]=1_6					,       __lowerCamelCase						:					List[str]=2					,       __lowerCamelCase						:					Optional[int]=4					,       __lowerCamelCase						:					str=4					,       __lowerCamelCase						:					Optional[int]="gelu"					,       __lowerCamelCase						:					Optional[int]=0.1					,       __lowerCamelCase						:					List[Any]=0.1					,       __lowerCamelCase						:					Any=2_0					,       __lowerCamelCase						:					Dict=2					,       __lowerCamelCase						:					Dict=1					,       __lowerCamelCase						:					List[Any]=0					,       __lowerCamelCase						:					Union[str, Any]=1_6					,       __lowerCamelCase						:					Union[str, Any]=1_6					,       )    ->   Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase							=     parent
         UpperCAmelCase							=     batch_size
         UpperCAmelCase							=     seq_length
         UpperCAmelCase							=     is_training
         UpperCAmelCase							=     use_labels
         UpperCAmelCase							=     vocab_size
         UpperCAmelCase							=     hidden_size
         UpperCAmelCase							=     num_hidden_layers
         UpperCAmelCase							=     num_attention_heads
         UpperCAmelCase							=     intermediate_size
         UpperCAmelCase							=     hidden_act
         UpperCAmelCase							=     hidden_dropout_prob
         UpperCAmelCase							=     attention_probs_dropout_prob
         UpperCAmelCase							=     max_position_embeddings
         UpperCAmelCase							=     eos_token_id
         UpperCAmelCase							=     pad_token_id
         UpperCAmelCase							=     bos_token_id
         UpperCAmelCase							=     embed_dim
         UpperCAmelCase							=     word_embed_proj_dim
         UpperCAmelCase							=     False





      def        _lowercase				(		self						:					Any       )    ->   str:






         """simple docstring"""







         UpperCAmelCase							=     ids_tensor([self.batch_size, self.seq_length - 1]					,       self.vocab_size       )
         UpperCAmelCase							=     tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size       )					,       1       )
         UpperCAmelCase							=     tf.concat([input_ids, eos_tensor]					,       axis=1       )

         UpperCAmelCase							=     self.config_cls(
             vocab_size=self.vocab_size					,       hidden_size=self.hidden_size					,       num_hidden_layers=self.num_hidden_layers					,       num_attention_heads=self.num_attention_heads					,       ffn_dim=self.intermediate_size					,       dropout=self.hidden_dropout_prob					,       attention_dropout=self.attention_probs_dropout_prob					,       max_position_embeddings=self.max_position_embeddings					,       eos_token_id=self.eos_token_id					,       bos_token_id=self.bos_token_id					,       pad_token_id=self.pad_token_id					,       embed_dim=self.embed_dim					,       word_embed_proj_dim=self.word_embed_proj_dim					,       is_encoder_decoder=__lowerCamelCase					,       **self.config_updates					,       )
         UpperCAmelCase							=     prepare_opt_inputs_dict(__lowerCamelCase					,       __lowerCamelCase       )
         return config, inputs_dict





      def        _lowercase				(		self						:					Optional[int]					,       __lowerCamelCase						:					Any					,       __lowerCamelCase						:					Dict       )    ->   int:






         """simple docstring"""







         UpperCAmelCase							=     TFOPTModel(config=__lowerCamelCase       )
         UpperCAmelCase							=     inputs_dict["""input_ids"""]

         UpperCAmelCase							=     input_ids[:1, :]
         UpperCAmelCase							=     inputs_dict["""attention_mask"""][:1, :]
         UpperCAmelCase							=     1

         # first forward pass
         UpperCAmelCase							=     model(__lowerCamelCase					,       attention_mask=__lowerCamelCase					,       use_cache=__lowerCamelCase       )

         UpperCAmelCase				,  UpperCAmelCase							=     outputs.to_tuple()

         # create hypothetical next token and extent to next_input_ids
         UpperCAmelCase							=     ids_tensor((self.batch_size, 3)					,       config.vocab_size       )
         UpperCAmelCase							=     tf.cast(ids_tensor((self.batch_size, 3)					,       2       )					,       tf.inta       )

         # append to next input_ids and
         UpperCAmelCase							=     tf.concat([input_ids, next_tokens]					,       axis=-1       )
         UpperCAmelCase							=     tf.concat([attention_mask, next_attn_mask]					,       axis=-1       )

         UpperCAmelCase							=     model(__lowerCamelCase					,       attention_mask=__lowerCamelCase       )[0]
         UpperCAmelCase							=     model(__lowerCamelCase					,       attention_mask=__lowerCamelCase					,       past_key_values=__lowerCamelCase       )[0]

         self.parent.assertEqual(next_tokens.shape[1]					,       output_from_past.shape[1]       )

         # select random slice
         UpperCAmelCase							=     int(ids_tensor((1,)					,       output_from_past.shape[-1]       )       )
         UpperCAmelCase							=     output_from_no_past[:, -3:, random_slice_idx]
         UpperCAmelCase							=     output_from_past[:, :, random_slice_idx]

         # test that outputs are equal for slice
         tf.debugging.assert_near(__lowerCamelCase					,       __lowerCamelCase					,       rtol=1e-3       )






@require_tf
class   __lowercase				(  __snake_case ,					__snake_case ,					unittest.TestCase  ):
      UpperCamelCase								=       (TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
      UpperCamelCase								=       (TFOPTForCausalLM,) if is_tf_available() else ()
      UpperCamelCase								=       (
          {'''feature-extraction''': TFOPTModel, '''text-generation''': TFOPTForCausalLM} if is_tf_available() else {}
      )
      UpperCamelCase								=       False
      UpperCamelCase								=       False
      UpperCamelCase								=       False
      UpperCamelCase								=       10





      def        _lowercase				(		self						:					str       )    ->   Dict:






         """simple docstring"""







         UpperCAmelCase							=     TFOPTModelTester(self       )
         UpperCAmelCase							=     ConfigTester(self					,       config_class=__lowerCamelCase       )





      def        _lowercase				(		self						:					Dict       )    ->   Tuple:






         """simple docstring"""







         self.config_tester.run_common_tests()





      def        _lowercase				(		self						:					Union[str, Any]       )    ->   Any:






         """simple docstring"""







         UpperCAmelCase							=     self.model_tester.prepare_config_and_inputs_for_common()
         self.model_tester.check_decoder_model_past_large_inputs(*__lowerCamelCase       )





      def        _lowercase				(		self						:					List[Any]       )    ->   Tuple:






         """simple docstring"""







         UpperCAmelCase				,  UpperCAmelCase							=     self.model_tester.prepare_config_and_inputs_for_common()

         def _get_word_embedding_weight(__lowerCamelCase						:					int					,       __lowerCamelCase						:					Tuple       ):
            if hasattr(__lowerCamelCase					,       """weight"""       ):
               return embedding_layer.weight
            else:
               # Here we build the word embeddings weights if not exists.
               # And then we retry to get the attribute once built.
               model.build()
               if hasattr(__lowerCamelCase					,       """weight"""       ):
                  return embedding_layer.weight
               else:
                  return None

         for model_class in self.all_model_classes:
            for size in [config.vocab_size - 1_0, config.vocab_size + 1_0]:
               # build the embeddings
               UpperCAmelCase							=     model_class(config=__lowerCamelCase       )
               UpperCAmelCase							=     _get_word_embedding_weight(__lowerCamelCase					,       model.get_input_embeddings()       )
               UpperCAmelCase							=     _get_word_embedding_weight(__lowerCamelCase					,       model.get_output_embeddings()       )

               # reshape the embeddings
               model.resize_token_embeddings(__lowerCamelCase       )
               UpperCAmelCase							=     _get_word_embedding_weight(__lowerCamelCase					,       model.get_input_embeddings()       )
               UpperCAmelCase							=     _get_word_embedding_weight(__lowerCamelCase					,       model.get_output_embeddings()       )

               # check that the resized embeddings size matches the desired size.
               UpperCAmelCase							=     size if size is not None else config.vocab_size

               self.assertEqual(new_input_embeddings.shape[0]					,       __lowerCamelCase       )

               # check that weights remain the same after resizing
               UpperCAmelCase							=     True
               for pa, pa in zip(old_input_embeddings.value()					,       new_input_embeddings.value()       ):
                  if tf.math.reduce_sum(tf.math.abs(pa - pa       )       ) > 0:
                     UpperCAmelCase							=     False
               self.assertTrue(__lowerCamelCase       )

               if old_output_embeddings is not None and new_output_embeddings is not None:
                  self.assertEqual(new_output_embeddings.shape[0]					,       __lowerCamelCase       )

                  UpperCAmelCase							=     True
                  for pa, pa in zip(old_output_embeddings.value()					,       new_output_embeddings.value()       ):
                     if tf.math.reduce_sum(tf.math.abs(pa - pa       )       ) > 0:
                        UpperCAmelCase							=     False
                  self.assertTrue(__lowerCamelCase       )

def   _UpperCamelCase				(						lowerCAmelCase_   )						->Optional[Any]:
   return tf.constant(lowerCAmelCase_  ,						dtype=tf.intaa   )






@require_tf
class   __lowercase				(  unittest.TestCase  ):
      UpperCamelCase								=       99





      def        _lowercase				(		self						:					Optional[Any]       )    ->   Tuple:






         """simple docstring"""







         UpperCAmelCase							=     tf.ones((4, 1)					,       dtype=tf.intaa       ) * 2
         UpperCAmelCase							=     tf.concat([ids_tensor((4, 6)					,       self.vocab_size - 3       ) + 3, eos_column_vector]					,       axis=1       )
         UpperCAmelCase							=     input_ids.shape[0]
         UpperCAmelCase							=     OPTConfig(
             vocab_size=self.vocab_size					,       hidden_size=2_4					,       num_hidden_layers=2					,       num_attention_heads=2					,       ffn_dim=3_2					,       max_position_embeddings=4_8					,       eos_token_id=2					,       pad_token_id=1					,       bos_token_id=0					,       )
         return config, input_ids, batch_size






@require_sentencepiece
@require_tf
class   __lowercase				(  unittest.TestCase  ):





      @slow
      def        _lowercase				(		self						:					Tuple       )    ->   Dict:






         """simple docstring"""







         UpperCAmelCase							=     TFOPTModel.from_pretrained("""facebook/opt-350m"""       )
         UpperCAmelCase							=     _long_tensor([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]       )
         UpperCAmelCase							=     tf.not_equal(__lowerCamelCase					,       model.config.pad_token_id       )
         with tf.GradientTape():
            UpperCAmelCase							=     model(input_ids=__lowerCamelCase					,       attention_mask=__lowerCamelCase       ).last_hidden_state
         UpperCAmelCase							=     (1, 1_1, 5_1_2)
         self.assertEqual(output.shape					,       __lowerCamelCase       )
         UpperCAmelCase							=     tf.constant(
             [[-0.2_873, -1.9_218, -0.3_033], [-1.2_710, -0.1_338, -0.1_902], [0.4_095, 0.1_214, -1.3_121]]       )
         self.assertTrue(np.allclose(output[:, :3, :3]					,       __lowerCamelCase					,       atol=4e-3       )       )

         UpperCAmelCase							=     tf.function(__lowerCamelCase					,       jit_compile=__lowerCamelCase       )
         UpperCAmelCase							=     xla_generate(__lowerCamelCase					,       __lowerCamelCase       )[0]
         self.assertTrue(np.allclose(output[:, :3, :3]					,       __lowerCamelCase					,       atol=4e-2       )       )






@require_tf
@slow
class   __lowercase				(  unittest.TestCase  ):





      def        _lowercase				(		self						:					Optional[int]       )    ->   List[Any]:






         """simple docstring"""







         super().setUp()
         UpperCAmelCase							=     """facebook/opt-350m"""





      def        _lowercase				(		self						:					Optional[int]       )    ->   List[Any]:






         """simple docstring"""







         UpperCAmelCase							=     TFOPTForCausalLM.from_pretrained(self.path_model       )
         UpperCAmelCase							=     GPTaTokenizer.from_pretrained(self.path_model       )

         UpperCAmelCase							=     [
             """Today is a beautiful day and I want to""",
             """In the city of""",
             """Paris is the capital of France and""",
             """Computers and mobile phones have taken""",
         ]
         # verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
         UpperCAmelCase							=     tokenizer(__lowerCamelCase					,       return_tensors="""tf"""					,       padding=__lowerCamelCase					,       add_special_tokens=__lowerCamelCase       )
         UpperCAmelCase							=     tf.math.reduce_mean(model(inputs.input_ids					,       attention_mask=inputs.attention_mask       )[0]					,       axis=-1       )
         UpperCAmelCase							=     tf.constant(
             [
                 [1.3_851, -13.8_923, -10.5_229, -10.7_533, -0.2_309, -10.2_384, -0.5_365, -9.0_947, -5.1_670],
                 [-4.7_073, -10.6_276, -3.9_415, -21.5_242, -0.2_822, -0.2_822, -0.2_822, -0.2_822, -0.2_822],
                 [0.6_247, -3.4_229, -8.9_179, -1.4_297, -14.1_650, 1.4_146, -9.0_218, -0.2_703, -0.2_703],
                 [6.4_783, -1.9_913, -10.7_926, -2.3_336, 1.5_092, -0.9_974, -6.8_213, 1.3_477, 1.3_477],
             ]       )
         self.assertTrue(np.allclose(__lowerCamelCase					,       __lowerCamelCase					,       atol=1e-4       )       )

         UpperCAmelCase							=     tf.function(__lowerCamelCase					,       jit_compile=__lowerCamelCase       )
         UpperCAmelCase							=     tf.math.reduce_mean(xla_generate(inputs.input_ids					,       attention_mask=inputs.attention_mask       )[0]					,       axis=-1       )
         self.assertTrue(np.allclose(__lowerCamelCase					,       __lowerCamelCase					,       atol=1e-4       )       )






@require_tf
@slow
class   __lowercase				(  unittest.TestCase  ):





      @property
      def        _lowercase				(		self						:					Any       )    ->   str:






         """simple docstring"""







         return [
             "Today is a beautiful day and I want",
             "In the city of",
             "Paris is the capital of France and",
             "Computers and mobile phones have taken",
         ]





      def        _lowercase				(		self						:					List[str]       )    ->   Tuple:






         """simple docstring"""







         UpperCAmelCase							=     """facebook/opt-125m"""

         UpperCAmelCase							=     [
             """Today is a beautiful day and I want to""",
             """In the city of New York, the city""",
             """Paris is the capital of France and the capital""",
             """Computers and mobile phones have taken over the""",
         ]

         UpperCAmelCase							=     []
         UpperCAmelCase							=     GPTaTokenizer.from_pretrained(__lowerCamelCase       )
         UpperCAmelCase							=     TFOPTForCausalLM.from_pretrained(__lowerCamelCase       )

         for prompt in self.prompts:
            UpperCAmelCase							=     tokenizer(__lowerCamelCase					,       return_tensors="""tf"""       ).input_ids

            UpperCAmelCase							=     model.generate(__lowerCamelCase					,       max_length=1_0       )

            UpperCAmelCase							=     tokenizer.batch_decode(__lowerCamelCase					,       skip_special_tokens=__lowerCamelCase       )
            predicted_outputs += generated_string

         self.assertListEqual(__lowerCamelCase					,       __lowerCamelCase       )





      def        _lowercase				(		self						:					Tuple       )    ->   str:






         """simple docstring"""







         UpperCAmelCase							=     """facebook/opt-350m"""

         UpperCAmelCase							=     GPTaTokenizer.from_pretrained(__lowerCamelCase       )
         UpperCAmelCase							=     TFOPTForCausalLM.from_pretrained(__lowerCamelCase       )

         UpperCAmelCase							=     """left"""

         # use different length sentences to test batching
         UpperCAmelCase							=     [
             """Hello, my dog is a little""",
             """Today, I""",
         ]

         UpperCAmelCase							=     tokenizer(__lowerCamelCase					,       return_tensors="""tf"""					,       padding=__lowerCamelCase       )
         UpperCAmelCase							=     inputs["""input_ids"""]

         UpperCAmelCase							=     model.generate(input_ids=__lowerCamelCase					,       attention_mask=inputs["""attention_mask"""]       )

         UpperCAmelCase							=     tokenizer(sentences[0]					,       return_tensors="""tf"""       ).input_ids
         UpperCAmelCase							=     model.generate(input_ids=__lowerCamelCase       )

         UpperCAmelCase							=     inputs_non_padded.shape[-1] - tf.math.reduce_sum(
             tf.cast(inputs["""attention_mask"""][-1]					,       tf.intaa       )       )
         UpperCAmelCase							=     tokenizer(sentences[1]					,       return_tensors="""tf"""       ).input_ids
         UpperCAmelCase							=     model.generate(input_ids=__lowerCamelCase					,       max_length=model.config.max_length - num_paddings       )

         UpperCAmelCase							=     tokenizer.batch_decode(__lowerCamelCase					,       skip_special_tokens=__lowerCamelCase       )
         UpperCAmelCase							=     tokenizer.decode(output_non_padded[0]					,       skip_special_tokens=__lowerCamelCase       )
         UpperCAmelCase							=     tokenizer.decode(output_padded[0]					,       skip_special_tokens=__lowerCamelCase       )

         UpperCAmelCase							=     [
             """Hello, my dog is a little bit of a dork.\nI'm a little bit""",
             """Today, I was in the middle of a conversation with a friend about the""",
         ]
         self.assertListEqual(__lowerCamelCase					,       __lowerCamelCase       )
         self.assertListEqual(__lowerCamelCase					,       [non_padded_sentence, padded_sentence]       )





      def        _lowercase				(		self						:					str       )    ->   Tuple:






         """simple docstring"""







         UpperCAmelCase							=     """facebook/opt-350m"""

         UpperCAmelCase							=     [
             """Today is a beautiful day and I want to""",
             """In the city of San Francisco, the city""",
             """Paris is the capital of France and the capital""",
             """Computers and mobile phones have taken over the""",
         ]

         UpperCAmelCase							=     []
         UpperCAmelCase							=     GPTaTokenizer.from_pretrained(__lowerCamelCase       )
         UpperCAmelCase							=     TFOPTForCausalLM.from_pretrained(__lowerCamelCase       )

         for prompt in self.prompts:
            UpperCAmelCase							=     tokenizer(__lowerCamelCase					,       return_tensors="""tf"""       ).input_ids

            UpperCAmelCase							=     model.generate(__lowerCamelCase					,       max_length=1_0       )

            UpperCAmelCase							=     tokenizer.batch_decode(__lowerCamelCase					,       skip_special_tokens=__lowerCamelCase       )
            predicted_outputs += generated_string

         self.assertListEqual(__lowerCamelCase					,       __lowerCamelCase       )







from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__a     =			"""\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

__a     =			"""\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

__a     =			"""
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_   )						->List[str]:
   return float((preds == labels).mean()   )
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_="binary"   )						->Union[str, Any]:
   UpperCAmelCase							=     simple_accuracy(lowerCAmelCase_  ,						lowerCAmelCase_   )
   UpperCAmelCase							=     float(fa_score(y_true=lowerCAmelCase_  ,						y_pred=lowerCAmelCase_  ,						average=lowerCAmelCase_   )   )
   return {
       "accuracy": acc,
       "f1": fa,
   }
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_   )						->List[Any]:
   UpperCAmelCase							=     {}
   for id_pred, label in zip(lowerCAmelCase_  ,						lowerCAmelCase_   ):
      UpperCAmelCase							=     F"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
      UpperCAmelCase							=     id_pred["""prediction"""]
      if question_id in question_map:
         question_map[question_id].append((pred, label)   )
      else:
         UpperCAmelCase							=     [(pred, label)]
   UpperCAmelCase				,  UpperCAmelCase							=     [], []
   for question, preds_labels in question_map.items():
      UpperCAmelCase				,  UpperCAmelCase							=     zip(*lowerCAmelCase_   )
      UpperCAmelCase							=     fa_score(y_true=lowerCAmelCase_  ,						y_pred=lowerCAmelCase_  ,						average="""macro"""   )
      fas.append(lowerCAmelCase_   )
      UpperCAmelCase							=     int(sum(pred == label for pred, label in preds_labels   ) == len(lowerCAmelCase_   )   )
      ems.append(lowerCAmelCase_   )
   UpperCAmelCase							=     float(sum(lowerCAmelCase_   ) / len(lowerCAmelCase_   )   )
   UpperCAmelCase							=     sum(lowerCAmelCase_   ) / len(lowerCAmelCase_   )
   UpperCAmelCase							=     float(fa_score(y_true=lowerCAmelCase_  ,						y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]   )   )
   return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,					_KWARGS_DESCRIPTION  )
class   __lowercase				(  datasets.Metric  ):





      def        _lowercase				(		self						:					int       )    ->   Any:






         """simple docstring"""







         if self.config_name not in [
             "boolq",
             "cb",
             "copa",
             "multirc",
             "record",
             "rte",
             "wic",
             "wsc",
             "wsc.fixed",
             "axb",
             "axg",
         ]:
            raise KeyError(
                """You should supply a configuration name selected in """
                """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""       )
         return datasets.MetricInfo(
             description=_DESCRIPTION					,       citation=_CITATION					,       inputs_description=_KWARGS_DESCRIPTION					,       features=datasets.Features(self._get_feature_types()       )					,       codebase_urls=[]					,       reference_urls=[]					,       format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None					,       )





      def        _lowercase				(		self						:					Optional[Any]       )    ->   Any:






         """simple docstring"""







         if self.config_name == "record":
            return {
                "predictions": {
                    "idx": {
                        "passage": datasets.Value("""int64"""       ),
                        "query": datasets.Value("""int64"""       ),
                    },
                    "prediction_text": datasets.Value("""string"""       ),
                },
                "references": {
                    "idx": {
                        "passage": datasets.Value("""int64"""       ),
                        "query": datasets.Value("""int64"""       ),
                    },
                    "answers": datasets.Sequence(datasets.Value("""string"""       )       ),
                },
            }
         elif self.config_name == "multirc":
            return {
                "predictions": {
                    "idx": {
                        "answer": datasets.Value("""int64"""       ),
                        "paragraph": datasets.Value("""int64"""       ),
                        "question": datasets.Value("""int64"""       ),
                    },
                    "prediction": datasets.Value("""int64"""       ),
                },
                "references": datasets.Value("""int64"""       ),
            }
         else:
            return {
                "predictions": datasets.Value("""int64"""       ),
                "references": datasets.Value("""int64"""       ),
            }





      def        _lowercase				(		self						:					Dict					,       __lowerCamelCase						:					int					,       __lowerCamelCase						:					Optional[int]       )    ->   List[Any]:






         """simple docstring"""







         if self.config_name == "axb":
            return {"matthews_correlation": matthews_corrcoef(__lowerCamelCase					,       __lowerCamelCase       )}
         elif self.config_name == "cb":
            return acc_and_fa(__lowerCamelCase					,       __lowerCamelCase					,       fa_avg="""macro"""       )
         elif self.config_name == "record":
            UpperCAmelCase							=     [
                {
                    """qas""": [
                        {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
                        for ref in references
                    ]
                }
            ]
            UpperCAmelCase							=     {pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
            return evaluate_record(__lowerCamelCase					,       __lowerCamelCase       )[0]
         elif self.config_name == "multirc":
            return evaluate_multirc(__lowerCamelCase					,       __lowerCamelCase       )
         elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
            return {"accuracy": simple_accuracy(__lowerCamelCase					,       __lowerCamelCase       )}
         else:
            raise KeyError(
                """You should supply a configuration name selected in """
                """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""       )



