


import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__a     =			"""▁"""

__a     =			get_tests_dir("""fixtures/test_sentencepiece.model""")






@require_sentencepiece
@require_tokenizers
class   __lowercase				(  __snake_case ,					unittest.TestCase  ):
      UpperCamelCase								=       BigBirdTokenizer
      UpperCamelCase								=       BigBirdTokenizerFast
      UpperCamelCase								=       True
      UpperCamelCase								=       True





      def        _lowercase				(		self						:					Any       )    ->   Optional[int]:






         """simple docstring"""







         super().setUp()

         UpperCAmelCase							=     self.tokenizer_class(__lowerCamelCase					,       keep_accents=__lowerCamelCase       )
         tokenizer.save_pretrained(self.tmpdirname       )





      def        _lowercase				(		self						:					int       )    ->   Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase							=     """<s>"""
         UpperCAmelCase							=     1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase       )					,       __lowerCamelCase       )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase       )					,       __lowerCamelCase       )





      def        _lowercase				(		self						:					str       )    ->   str:






         """simple docstring"""







         UpperCAmelCase							=     list(self.get_tokenizer().get_vocab().keys()       )

         self.assertEqual(vocab_keys[0]					,       """<unk>"""       )
         self.assertEqual(vocab_keys[1]					,       """<s>"""       )
         self.assertEqual(vocab_keys[-1]					,       """[MASK]"""       )
         self.assertEqual(len(__lowerCamelCase       )					,       1_0_0_4       )





      def        _lowercase				(		self						:					Optional[int]       )    ->   Optional[Any]:






         """simple docstring"""







         self.assertEqual(self.get_tokenizer().vocab_size					,       1_0_0_0       )





      def        _lowercase				(		self						:					Dict       )    ->   List[Any]:






         """simple docstring"""







         if not self.test_rust_tokenizer:
            return

         UpperCAmelCase							=     self.get_tokenizer()
         UpperCAmelCase							=     self.get_rust_tokenizer()

         UpperCAmelCase							=     """I was born in 92000, and this is falsé."""

         UpperCAmelCase							=     tokenizer.tokenize(__lowerCamelCase       )
         UpperCAmelCase							=     rust_tokenizer.tokenize(__lowerCamelCase       )
         self.assertListEqual(__lowerCamelCase					,       __lowerCamelCase       )

         UpperCAmelCase							=     tokenizer.encode(__lowerCamelCase					,       add_special_tokens=__lowerCamelCase       )
         UpperCAmelCase							=     rust_tokenizer.encode(__lowerCamelCase					,       add_special_tokens=__lowerCamelCase       )
         self.assertListEqual(__lowerCamelCase					,       __lowerCamelCase       )

         UpperCAmelCase							=     self.get_rust_tokenizer()
         UpperCAmelCase							=     tokenizer.encode(__lowerCamelCase       )
         UpperCAmelCase							=     rust_tokenizer.encode(__lowerCamelCase       )
         self.assertListEqual(__lowerCamelCase					,       __lowerCamelCase       )





      def        _lowercase				(		self						:					str       )    ->   Any:






         """simple docstring"""







         UpperCAmelCase							=     BigBirdTokenizer(__lowerCamelCase					,       keep_accents=__lowerCamelCase       )

         UpperCAmelCase							=     tokenizer.tokenize("""This is a test"""       )
         self.assertListEqual(__lowerCamelCase					,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]       )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(__lowerCamelCase       )					,       [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]					,       )

         UpperCAmelCase							=     tokenizer.tokenize("""I was born in 92000, and this is falsé."""       )
         self.assertListEqual(
             __lowerCamelCase					,       [
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """9""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """é""",
                 """.""",
             ]					,       )
         UpperCAmelCase							=     tokenizer.convert_tokens_to_ids(__lowerCamelCase       )
         self.assertListEqual(
             __lowerCamelCase					,       [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4]					,       )

         UpperCAmelCase							=     tokenizer.convert_ids_to_tokens(__lowerCamelCase       )
         self.assertListEqual(
             __lowerCamelCase					,       [
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """<unk>""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """<unk>""",
                 """.""",
             ]					,       )





      @cached_property
      def        _lowercase				(		self						:					str       )    ->   Optional[int]:






         """simple docstring"""







         return BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base"""       )





      @slow
      def        _lowercase				(		self						:					List[str]       )    ->   Optional[int]:






         """simple docstring"""







         UpperCAmelCase							=     """Hello World!"""
         UpperCAmelCase							=     [6_5, 1_8_5_3_6, 2_2_6_0, 1_0_1, 6_6]

         self.assertListEqual(__lowerCamelCase					,       self.big_tokenizer.encode(__lowerCamelCase       )       )





      @slow
      def        _lowercase				(		self						:					int       )    ->   Optional[Any]:






         """simple docstring"""







         UpperCAmelCase							=     (
             """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
             """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
         )
         # fmt: off
         UpperCAmelCase							=     [6_5, 8_7_1, 4_1_9, 3_5_8, 9_4_6, 9_9_1, 2_5_2_1, 4_5_2, 3_5_8, 1_3_5_7, 3_8_7, 7_7_5_1, 3_5_3_6, 1_1_2, 9_8_5, 4_5_6, 1_2_6, 8_6_5, 9_3_8, 5_4_0_0, 5_7_3_4, 4_5_8, 1_3_6_8, 4_6_7, 7_8_6, 2_4_6_2, 5_2_4_6, 1_1_5_9, 6_3_3, 8_6_5, 4_5_1_9, 4_5_7, 5_8_2, 8_5_2, 2_5_5_7, 4_2_7, 9_1_6, 5_0_8, 4_0_5, 3_4_3_2_4, 4_9_7, 3_9_1, 4_0_8, 1_1_3_4_2, 1_2_4_4, 3_8_5, 1_0_0, 9_3_8, 9_8_5, 4_5_6, 5_7_4, 3_6_2, 1_2_5_9_7, 3_2_0_0, 3_1_2_9, 1_1_7_2, 6_6]  # noqa: E231
         # fmt: on
         self.assertListEqual(__lowerCamelCase					,       self.big_tokenizer.encode(__lowerCamelCase       )       )





      @require_torch
      @slow
      def        _lowercase				(		self						:					List[Any]       )    ->   List[Any]:






         """simple docstring"""







         import torch

         from transformers import BigBirdConfig, BigBirdModel

         # Build sequence
         UpperCAmelCase							=     list(self.big_tokenizer.get_vocab().keys()       )[:1_0]
         UpperCAmelCase							=     """ """.join(__lowerCamelCase       )
         UpperCAmelCase							=     self.big_tokenizer.encode_plus(__lowerCamelCase					,       return_tensors="""pt"""					,       return_token_type_ids=__lowerCamelCase       )
         UpperCAmelCase							=     self.big_tokenizer.batch_encode_plus(
             [sequence + """ """ + sequence]					,       return_tensors="""pt"""					,       return_token_type_ids=__lowerCamelCase       )

         UpperCAmelCase							=     BigBirdConfig(attention_type="""original_full"""       )
         UpperCAmelCase							=     BigBirdModel(__lowerCamelCase       )

         assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

         with torch.no_grad():
            model(**__lowerCamelCase       )
            model(**__lowerCamelCase       )





      @slow
      def        _lowercase				(		self						:					Dict       )    ->   Dict:






         """simple docstring"""







         UpperCAmelCase							=     BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base"""       )
         UpperCAmelCase							=     tokenizer.decode(tokenizer("""Paris is the [MASK]."""       ).input_ids       )

         self.assertTrue(decoded_text == """[CLS] Paris is the[MASK].[SEP]"""       )





      @slow
      def        _lowercase				(		self						:					Optional[Any]       )    ->   Any:






         """simple docstring"""







         UpperCAmelCase							=     {"""input_ids""": [[6_5, 3_9_2_8_6, 4_5_8, 3_6_3_3_5, 2_0_0_1, 4_5_6, 1_3_0_7_3, 1_3_2_6_6, 4_5_5, 1_1_3, 7_7_4_6, 1_7_4_1, 1_1_1_5_7, 3_9_1, 1_3_0_7_3, 1_3_2_6_6, 4_5_5, 1_1_3, 3_9_6_7, 3_5_4_1_2, 1_1_3, 4_9_3_6, 1_0_9, 3_8_7_0, 2_3_7_7, 1_1_3, 3_0_0_8_4, 4_5_7_2_0, 4_5_8, 1_3_4, 1_7_4_9_6, 1_1_2, 5_0_3, 1_1_6_7_2, 1_1_3, 1_1_8, 1_1_2, 5_6_6_5, 1_3_3_4_7, 3_8_6_8_7, 1_1_2, 1_4_9_6, 3_1_3_8_9, 1_1_2, 3_2_6_8, 4_7_2_6_4, 1_3_4, 9_6_2, 1_1_2, 1_6_3_7_7, 8_0_3_5, 2_3_1_3_0, 4_3_0, 1_2_1_6_9, 1_5_5_1_8, 2_8_5_9_2, 4_5_8, 1_4_6, 4_1_6_9_7, 1_0_9, 3_9_1, 1_2_1_6_9, 1_5_5_1_8, 1_6_6_8_9, 4_5_8, 1_4_6, 4_1_3_5_8, 1_0_9, 4_5_2, 7_2_6, 4_0_3_4, 1_1_1, 7_6_3, 3_5_4_1_2, 5_0_8_2, 3_8_8, 1_9_0_3, 1_1_1, 9_0_5_1, 3_9_1, 2_8_7_0, 4_8_9_1_8, 1_9_0_0, 1_1_2_3, 5_5_0, 9_9_8, 1_1_2, 9_5_8_6, 1_5_9_8_5, 4_5_5, 3_9_1, 4_1_0, 2_2_9_5_5, 3_7_6_3_6, 1_1_4, 6_6], [6_5, 4_4_8, 1_7_4_9_6, 4_1_9, 3_6_6_3, 3_8_5, 7_6_3, 1_1_3, 2_7_5_3_3, 2_8_7_0, 3_2_8_3, 1_3_0_4_3, 1_6_3_9, 2_4_7_1_3, 5_2_3, 6_5_6, 2_4_0_1_3, 1_8_5_5_0, 2_5_2_1, 5_1_7, 2_7_0_1_4, 2_1_2_4_4, 4_2_0, 1_2_1_2, 1_4_6_5, 3_9_1, 9_2_7, 4_8_3_3, 3_8_8, 5_7_8, 1_1_7_8_6, 1_1_4, 6_6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [6_5, 4_8_4, 2_1_6_9, 7_6_8_7, 2_1_9_3_2, 1_8_1_4_6, 7_2_6, 3_6_3, 1_7_0_3_2, 3_3_9_1, 1_1_4, 6_6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=__lowerCamelCase					,       model_name="""google/bigbird-roberta-base"""					,       revision="""215c99f1600e06f83acce68422f2035b2b5c3510"""					,       )







from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__a     =			"""\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

__a     =			"""\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

__a     =			"""
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_   )						->List[str]:
   return float((preds == labels).mean()   )
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_="binary"   )						->Union[str, Any]:
   UpperCAmelCase							=     simple_accuracy(lowerCAmelCase_  ,						lowerCAmelCase_   )
   UpperCAmelCase							=     float(fa_score(y_true=lowerCAmelCase_  ,						y_pred=lowerCAmelCase_  ,						average=lowerCAmelCase_   )   )
   return {
       "accuracy": acc,
       "f1": fa,
   }
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_   )						->List[Any]:
   UpperCAmelCase							=     {}
   for id_pred, label in zip(lowerCAmelCase_  ,						lowerCAmelCase_   ):
      UpperCAmelCase							=     F"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
      UpperCAmelCase							=     id_pred["""prediction"""]
      if question_id in question_map:
         question_map[question_id].append((pred, label)   )
      else:
         UpperCAmelCase							=     [(pred, label)]
   UpperCAmelCase				,  UpperCAmelCase							=     [], []
   for question, preds_labels in question_map.items():
      UpperCAmelCase				,  UpperCAmelCase							=     zip(*lowerCAmelCase_   )
      UpperCAmelCase							=     fa_score(y_true=lowerCAmelCase_  ,						y_pred=lowerCAmelCase_  ,						average="""macro"""   )
      fas.append(lowerCAmelCase_   )
      UpperCAmelCase							=     int(sum(pred == label for pred, label in preds_labels   ) == len(lowerCAmelCase_   )   )
      ems.append(lowerCAmelCase_   )
   UpperCAmelCase							=     float(sum(lowerCAmelCase_   ) / len(lowerCAmelCase_   )   )
   UpperCAmelCase							=     sum(lowerCAmelCase_   ) / len(lowerCAmelCase_   )
   UpperCAmelCase							=     float(fa_score(y_true=lowerCAmelCase_  ,						y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]   )   )
   return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,					_KWARGS_DESCRIPTION  )
class   __lowercase				(  datasets.Metric  ):





      def        _lowercase				(		self						:					int       )    ->   Any:






         """simple docstring"""







         if self.config_name not in [
             "boolq",
             "cb",
             "copa",
             "multirc",
             "record",
             "rte",
             "wic",
             "wsc",
             "wsc.fixed",
             "axb",
             "axg",
         ]:
            raise KeyError(
                """You should supply a configuration name selected in """
                """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""       )
         return datasets.MetricInfo(
             description=_DESCRIPTION					,       citation=_CITATION					,       inputs_description=_KWARGS_DESCRIPTION					,       features=datasets.Features(self._get_feature_types()       )					,       codebase_urls=[]					,       reference_urls=[]					,       format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None					,       )





      def        _lowercase				(		self						:					Optional[Any]       )    ->   Any:






         """simple docstring"""







         if self.config_name == "record":
            return {
                "predictions": {
                    "idx": {
                        "passage": datasets.Value("""int64"""       ),
                        "query": datasets.Value("""int64"""       ),
                    },
                    "prediction_text": datasets.Value("""string"""       ),
                },
                "references": {
                    "idx": {
                        "passage": datasets.Value("""int64"""       ),
                        "query": datasets.Value("""int64"""       ),
                    },
                    "answers": datasets.Sequence(datasets.Value("""string"""       )       ),
                },
            }
         elif self.config_name == "multirc":
            return {
                "predictions": {
                    "idx": {
                        "answer": datasets.Value("""int64"""       ),
                        "paragraph": datasets.Value("""int64"""       ),
                        "question": datasets.Value("""int64"""       ),
                    },
                    "prediction": datasets.Value("""int64"""       ),
                },
                "references": datasets.Value("""int64"""       ),
            }
         else:
            return {
                "predictions": datasets.Value("""int64"""       ),
                "references": datasets.Value("""int64"""       ),
            }





      def        _lowercase				(		self						:					Dict					,       __lowerCamelCase						:					int					,       __lowerCamelCase						:					Optional[int]       )    ->   List[Any]:






         """simple docstring"""







         if self.config_name == "axb":
            return {"matthews_correlation": matthews_corrcoef(__lowerCamelCase					,       __lowerCamelCase       )}
         elif self.config_name == "cb":
            return acc_and_fa(__lowerCamelCase					,       __lowerCamelCase					,       fa_avg="""macro"""       )
         elif self.config_name == "record":
            UpperCAmelCase							=     [
                {
                    """qas""": [
                        {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
                        for ref in references
                    ]
                }
            ]
            UpperCAmelCase							=     {pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
            return evaluate_record(__lowerCamelCase					,       __lowerCamelCase       )[0]
         elif self.config_name == "multirc":
            return evaluate_multirc(__lowerCamelCase					,       __lowerCamelCase       )
         elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
            return {"accuracy": simple_accuracy(__lowerCamelCase					,       __lowerCamelCase       )}
         else:
            raise KeyError(
                """You should supply a configuration name selected in """
                """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""       )



