"""simple docstring"""





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


_UpperCamelCase     :		Any											=					"\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

_UpperCamelCase     :		Any											=					"\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

_UpperCamelCase     :		Any											=					"\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"





def        a_  (   _lowerCAmelCase						:				Dict     ,				_lowerCAmelCase						:				Optional[Any]  )     ->   str:





       '''simple docstring'''

       return float((preds == labels).mean()  )





def        a_  (   _lowerCAmelCase						:				Dict     ,				_lowerCAmelCase						:				str     ,				_lowerCAmelCase						:				Union[str, Any]="binary"  )     ->   Tuple:





       '''simple docstring'''

       lowercase__    :	Any     					=			simple_accuracy(_lowerCAmelCase     ,				_lowerCAmelCase  )
       lowercase__    :	str     					=			float(fa_score(y_true=_lowerCAmelCase     ,				y_pred=_lowerCAmelCase     ,				average=_lowerCAmelCase  )  )
       return {
           "accuracy": acc,
           "f1": fa,
       }





def        a_  (   _lowerCAmelCase						:				Optional[int]     ,				_lowerCAmelCase						:				Dict  )     ->   Union[str, Any]:





       '''simple docstring'''

       lowercase__    :	List[Any]     					=			{}
       for id_pred, label in zip(_lowerCAmelCase     ,				_lowerCAmelCase  ):
              lowercase__    :	str     					=			f"""{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}"""
              lowercase__    :	Optional[Any]     					=			id_pred['prediction']
              if question_id in question_map:
                     question_map[question_id].append((pred, label)  )
              else:
                     lowercase__    :	Optional[Any]     					=			[(pred, label)]
       lowercase__    :	List[Any]     					=			[], []
       for question, preds_labels in question_map.items():
              lowercase__    :	Dict     					=			zip(*_lowerCAmelCase  )
              lowercase__    :	Optional[Any]     					=			fa_score(y_true=_lowerCAmelCase     ,				y_pred=_lowerCAmelCase     ,				average='macro'  )
              fas.append(_lowerCAmelCase  )
              lowercase__    :	Any     					=			int(sum(pred == label for pred, label in preds_labels  ) == len(_lowerCAmelCase  )  )
              ems.append(_lowerCAmelCase  )
       lowercase__    :	Union[str, Any]     					=			float(sum(_lowerCAmelCase  ) / len(_lowerCAmelCase  )  )
       lowercase__    :	Dict     					=			sum(_lowerCAmelCase  ) / len(_lowerCAmelCase  )
       lowercase__    :	Tuple     					=			float(fa_score(y_true=_lowerCAmelCase     ,				y_pred=[id_pred['prediction'] for id_pred in ids_preds]  )  )
       return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION				,							_KWARGS_DESCRIPTION)
class   UpperCAmelCase_			(					datasets.Metric):


      def 	_UpperCAmelCase (     self							) ->      Union[str, Any]:
             if self.config_name not in [
                 "boolq",
                 "cb",
                 "copa",
                 "multirc",
                 "record",
                 "rte",
                 "wic",
                 "wsc",
                 "wsc.fixed",
                 "axb",
                 "axg",
             ]:
                    raise KeyError(
                        'You should supply a configuration name selected in '
                        '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'							)
             return datasets.MetricInfo(
                 description=_DESCRIPTION	,    citation=_CITATION	,    inputs_description=_KWARGS_DESCRIPTION	,    features=datasets.Features(self._get_feature_types()							)	,    codebase_urls=[]	,    reference_urls=[]	,    format='numpy' if not self.config_name == 'record' and not self.config_name == 'multirc' else None	,    )


      def 	_UpperCAmelCase (     self							) ->      Dict:
             if self.config_name == "record":
                    return {
                        "predictions": {
                            "idx": {
                                "passage": datasets.Value('int64'							),
                                "query": datasets.Value('int64'							),
                            },
                            "prediction_text": datasets.Value('string'							),
                        },
                        "references": {
                            "idx": {
                                "passage": datasets.Value('int64'							),
                                "query": datasets.Value('int64'							),
                            },
                            "answers": datasets.Sequence(datasets.Value('string'							)							),
                        },
                    }
             elif self.config_name == "multirc":
                    return {
                        "predictions": {
                            "idx": {
                                "answer": datasets.Value('int64'							),
                                "paragraph": datasets.Value('int64'							),
                                "question": datasets.Value('int64'							),
                            },
                            "prediction": datasets.Value('int64'							),
                        },
                        "references": datasets.Value('int64'							),
                    }
             else:
                    return {
                        "predictions": datasets.Value('int64'							),
                        "references": datasets.Value('int64'							),
                    }


      def 	_UpperCAmelCase (     self	,    a	,    a							) ->      Union[str, Any]:
             if self.config_name == "axb":
                    return {"matthews_correlation": matthews_corrcoef(a	,    a							)}
             elif self.config_name == "cb":
                    return acc_and_fa(a	,    a	,    fa_avg='macro'							)
             elif self.config_name == "record":
                    lowercase__    :	str     					=			[
                        {
                            'qas': [
                                {'id': ref['idx']['query'], 'answers': [{'text': ans} for ans in ref['answers']]}
                                for ref in references
                            ]
                        }
                    ]
                    lowercase__    :	str     					=			{pred['idx']['query']: pred['prediction_text'] for pred in predictions}
                    return evaluate_record(a	,    a							)[0]
             elif self.config_name == "multirc":
                    return evaluate_multirc(a	,    a							)
             elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
                    return {"accuracy": simple_accuracy(a	,    a							)}
             else:
                    raise KeyError(
                        'You should supply a configuration name selected in '
                        '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'							)


"""simple docstring"""





import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
       from transformers.trainer import DEFAULT_CALLBACKS

       from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel


class   UpperCAmelCase_			(					_a):


      def __init__(     self							) ->      Any:
             lowercase__    :	Tuple     					=			[]


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    **a							) ->      Any:
             self.events.append('on_init_end'							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    **a							) ->      Optional[int]:
             self.events.append('on_train_begin'							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    **a							) ->      List[str]:
             self.events.append('on_train_end'							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    **a							) ->      int:
             self.events.append('on_epoch_begin'							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    **a							) ->      Optional[Any]:
             self.events.append('on_epoch_end'							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    **a							) ->      int:
             self.events.append('on_step_begin'							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    **a							) ->      str:
             self.events.append('on_step_end'							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    **a							) ->      int:
             self.events.append('on_evaluate'							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    **a							) ->      Tuple:
             self.events.append('on_predict'							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    **a							) ->      Union[str, Any]:
             self.events.append('on_save'							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    **a							) ->      List[str]:
             self.events.append('on_log'							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    **a							) ->      Any:
             self.events.append('on_prediction_step'							)





@require_torch
class   UpperCAmelCase_			(					unittest.TestCase):


      def 	_UpperCAmelCase (     self							) ->      str:
             lowercase__    :	str     					=			tempfile.mkdtemp()


      def 	_UpperCAmelCase (     self							) ->      Dict:
             shutil.rmtree(self.output_dir							)


      def 	_UpperCAmelCase (     self	,    a=0	,    a=0	,    a=6_4	,    a=6_4	,    a=None	,    a=False	,    **a							) ->      int:
             # disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
             # its set to False since the tests later on depend on its value.
             lowercase__    :	str     					=			RegressionDataset(length=a							)
             lowercase__    :	Any     					=			RegressionDataset(length=a							)
             lowercase__    :	Optional[Any]     					=			RegressionModelConfig(a=a	,    b=a							)
             lowercase__    :	Union[str, Any]     					=			RegressionPreTrainedModel(a							)

             lowercase__    :	Tuple     					=			TrainingArguments(self.output_dir	,    disable_tqdm=a	,    report_to=[]	,    **a							)
             return Trainer(
                 a	,    a	,    train_dataset=a	,    eval_dataset=a	,    callbacks=a	,    )


      def 	_UpperCAmelCase (     self	,    a	,    a							) ->      Union[str, Any]:
             self.assertEqual(len(a							)	,    len(a							)							)

             # Order doesn't matter
             lowercase__    :	Optional[int]     					=			sorted(a	,    key=lambda a							: cb.__name__ if isinstance(a	,    a							) else cb.__class__.__name__							)
             lowercase__    :	Tuple     					=			sorted(a	,    key=lambda a							: cb.__name__ if isinstance(a	,    a							) else cb.__class__.__name__							)

             for cba, cba in zip(a	,    a							):
                    if isinstance(a	,    a							) and isinstance(a	,    a							):
                           self.assertEqual(a	,    a							)
                    elif isinstance(a	,    a							) and not isinstance(a	,    a							):
                           self.assertEqual(a	,    cba.__class__							)
                    elif not isinstance(a	,    a							) and isinstance(a	,    a							):
                           self.assertEqual(cba.__class__	,    a							)
                    else:
                           self.assertEqual(a	,    a							)


      def 	_UpperCAmelCase (     self	,    a							) ->      Optional[Any]:
             lowercase__    :	Dict     					=			['on_init_end', 'on_train_begin']
             lowercase__    :	List[Any]     					=			0
             lowercase__    :	Optional[int]     					=			len(trainer.get_eval_dataloader()							)
             lowercase__    :	Tuple     					=			['on_prediction_step'] * len(trainer.get_eval_dataloader()							) + ['on_log', 'on_evaluate']
             for _ in range(trainer.state.num_train_epochs							):
                    expected_events.append('on_epoch_begin'							)
                    for _ in range(a							):
                           step += 1
                           expected_events += ["on_step_begin", "on_step_end"]
                           if step % trainer.args.logging_steps == 0:
                                  expected_events.append('on_log'							)
                           if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
                                  expected_events += evaluation_events.copy()
                           if step % trainer.args.save_steps == 0:
                                  expected_events.append('on_save'							)
                    expected_events.append('on_epoch_end'							)
                    if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
                           expected_events += evaluation_events.copy()
             expected_events += ["on_log", "on_train_end"]
             return expected_events


      def 	_UpperCAmelCase (     self							) ->      Union[str, Any]:
             lowercase__    :	int     					=			self.get_trainer()
             lowercase__    :	str     					=			DEFAULT_CALLBACKS.copy() + [ProgressCallback]
             self.check_callbacks_equality(trainer.callback_handler.callbacks	,    a							)

             # Callbacks passed at init are added to the default callbacks
             lowercase__    :	str     					=			self.get_trainer(callbacks=[MyTestTrainerCallback]							)
             expected_callbacks.append(a							)
             self.check_callbacks_equality(trainer.callback_handler.callbacks	,    a							)

             # TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
             lowercase__    :	List[Any]     					=			self.get_trainer(disable_tqdm=a							)
             lowercase__    :	Optional[Any]     					=			DEFAULT_CALLBACKS.copy() + [PrinterCallback]
             self.check_callbacks_equality(trainer.callback_handler.callbacks	,    a							)


      def 	_UpperCAmelCase (     self							) ->      Any:
             lowercase__    :	int     					=			DEFAULT_CALLBACKS.copy() + [ProgressCallback]
             lowercase__    :	List[str]     					=			self.get_trainer()

             # We can add, pop, or remove by class name
             trainer.remove_callback(a							)
             expected_callbacks.remove(a							)
             self.check_callbacks_equality(trainer.callback_handler.callbacks	,    a							)

             lowercase__    :	Optional[Any]     					=			self.get_trainer()
             lowercase__    :	List[Any]     					=			trainer.pop_callback(a							)
             self.assertEqual(cb.__class__	,    a							)
             self.check_callbacks_equality(trainer.callback_handler.callbacks	,    a							)

             trainer.add_callback(a							)
             expected_callbacks.insert(0	,    a							)
             self.check_callbacks_equality(trainer.callback_handler.callbacks	,    a							)

             # We can also add, pop, or remove by instance
             lowercase__    :	int     					=			self.get_trainer()
             lowercase__    :	List[str]     					=			trainer.callback_handler.callbacks[0]
             trainer.remove_callback(a							)
             expected_callbacks.remove(a							)
             self.check_callbacks_equality(trainer.callback_handler.callbacks	,    a							)

             lowercase__    :	Tuple     					=			self.get_trainer()
             lowercase__    :	Dict     					=			trainer.callback_handler.callbacks[0]
             lowercase__    :	Union[str, Any]     					=			trainer.pop_callback(a							)
             self.assertEqual(a	,    a							)
             self.check_callbacks_equality(trainer.callback_handler.callbacks	,    a							)

             trainer.add_callback(a							)
             expected_callbacks.insert(0	,    a							)
             self.check_callbacks_equality(trainer.callback_handler.callbacks	,    a							)


      def 	_UpperCAmelCase (     self							) ->      Tuple:
             import warnings

             # XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
             warnings.simplefilter(action='ignore'	,    category=a							)

             lowercase__    :	Optional[Any]     					=			self.get_trainer(callbacks=[MyTestTrainerCallback]							)
             trainer.train()
             lowercase__    :	Any     					=			trainer.callback_handler.callbacks[-2].events
             self.assertEqual(a	,    self.get_expected_events(a							)							)

             # Independent log/save/eval
             lowercase__    :	List[str]     					=			self.get_trainer(callbacks=[MyTestTrainerCallback]	,    logging_steps=5							)
             trainer.train()
             lowercase__    :	int     					=			trainer.callback_handler.callbacks[-2].events
             self.assertEqual(a	,    self.get_expected_events(a							)							)

             lowercase__    :	Union[str, Any]     					=			self.get_trainer(callbacks=[MyTestTrainerCallback]	,    save_steps=5							)
             trainer.train()
             lowercase__    :	Union[str, Any]     					=			trainer.callback_handler.callbacks[-2].events
             self.assertEqual(a	,    self.get_expected_events(a							)							)

             lowercase__    :	List[str]     					=			self.get_trainer(callbacks=[MyTestTrainerCallback]	,    eval_steps=5	,    evaluation_strategy='steps'							)
             trainer.train()
             lowercase__    :	Optional[int]     					=			trainer.callback_handler.callbacks[-2].events
             self.assertEqual(a	,    self.get_expected_events(a							)							)

             lowercase__    :	int     					=			self.get_trainer(callbacks=[MyTestTrainerCallback]	,    evaluation_strategy='epoch'							)
             trainer.train()
             lowercase__    :	str     					=			trainer.callback_handler.callbacks[-2].events
             self.assertEqual(a	,    self.get_expected_events(a							)							)

             # A bit of everything
             lowercase__    :	Any     					=			self.get_trainer(
                 callbacks=[MyTestTrainerCallback]	,    logging_steps=3	,    save_steps=1_0	,    eval_steps=5	,    evaluation_strategy='steps'	,    )
             trainer.train()
             lowercase__    :	Any     					=			trainer.callback_handler.callbacks[-2].events
             self.assertEqual(a	,    self.get_expected_events(a							)							)

             # warning should be emitted for duplicated callbacks
             with patch('transformers.trainer_callback.logger.warning'							) as warn_mock:
                    lowercase__    :	str     					=			self.get_trainer(
                        callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]	,    )
                    assert str(a							) in warn_mock.call_args[0][0]

