






from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


A__			:    str         =				'\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n'

A__			:    str         =				'\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n'

A__			:    Union[str, Any]         =				'\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for \'record\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'prediction_text\': the predicted answer text\n        - for \'multirc\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question-answer pair as specified by the dataset\n            - \'prediction\': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for \'record\': list of question-answers dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'answers\': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for \'record\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1\': F1 score\n    - for \'multirc\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1_m\': Per-question macro-F1 score\n        - \'f1_a\': Average F1 score over all answers\n    - for \'axb\':\n        \'matthews_correlation\': Matthew Correlation\n    - for \'cb\':\n        - \'accuracy\': Accuracy\n        - \'f1\': F1 score\n    - for all others:\n        - \'accuracy\': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')\n    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]\n    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')\n    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'






def 						a    (		lowerCamelCase_	,     lowerCamelCase_       ):






				'''simple docstring'''
				return float((preds == labels).mean()       )






def 						a    (		lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_="binary"       ):






				'''simple docstring'''
				lowercase__     				=    simple_accuracy(lowercase_	,     lowercase_       )
				lowercase__     				=    float(fa_score(y_true=lowercase_	,     y_pred=lowercase_	,     average=lowercase_       )       )
				return {
				    "accuracy": acc,
				    "f1": fa,
				}






def 						a    (		lowerCamelCase_	,     lowerCamelCase_       ):






				'''simple docstring'''
				lowercase__     				=    {}
				for id_pred, label in zip(lowercase_	,     lowercase_       ):
								lowercase__     				=    F"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
								lowercase__     				=    id_pred["prediction"]
								if question_id in question_map:
												question_map[question_id].append((pred, label)       )
								else:
												lowercase__     				=    [(pred, label)]
				lowercase__     				=    [], []
				for question, preds_labels in question_map.items():
								lowercase__     				=    zip(*lowercase_       )
								lowercase__     				=    fa_score(y_true=lowercase_	,     y_pred=lowercase_	,     average='''macro'''       )
								fas.append(lowercase_       )
								lowercase__     				=    int(sum(pred == label for pred, label in preds_labels       ) == len(lowercase_       )       )
								ems.append(lowercase_       )
				lowercase__     				=    float(sum(lowercase_       ) / len(lowercase_       )       )
				lowercase__     				=    sum(lowercase_       ) / len(lowercase_       )
				lowercase__     				=    float(fa_score(y_true=lowercase_	,     y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]       )       )
				return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,_KWARGS_DESCRIPTION			)
class    _UpperCAmelCase      ( datasets.Metric			):

				"""simple docstring"""




				def        lowercase__     (  self		:					Dict						):

								'''simple docstring'''
								if self.config_name not in [
								    "boolq",
								    "cb",
								    "copa",
								    "multirc",
								    "record",
								    "rte",
								    "wic",
								    "wsc",
								    "wsc.fixed",
								    "axb",
								    "axg",
								]:
												raise KeyError(
												    '''You should supply a configuration name selected in '''
												    '''[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]'''						)
								return datasets.MetricInfo(
								    description=_DESCRIPTION,						citation=_CITATION,						inputs_description=_KWARGS_DESCRIPTION,						features=datasets.Features(self._get_feature_types()						),						codebase_urls=[],						reference_urls=[],						format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None,						)




				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								if self.config_name == "record":
												return {
												    "predictions": {
												        "idx": {
												            "passage": datasets.Value('''int64'''						),
												            "query": datasets.Value('''int64'''						),
												        },
												        "prediction_text": datasets.Value('''string'''						),
												    },
												    "references": {
												        "idx": {
												            "passage": datasets.Value('''int64'''						),
												            "query": datasets.Value('''int64'''						),
												        },
												        "answers": datasets.Sequence(datasets.Value('''string'''						)						),
												    },
												}
								elif self.config_name == "multirc":
												return {
												    "predictions": {
												        "idx": {
												            "answer": datasets.Value('''int64'''						),
												            "paragraph": datasets.Value('''int64'''						),
												            "question": datasets.Value('''int64'''						),
												        },
												        "prediction": datasets.Value('''int64'''						),
												    },
												    "references": datasets.Value('''int64'''						),
												}
								else:
												return {
												    "predictions": datasets.Value('''int64'''						),
												    "references": datasets.Value('''int64'''						),
												}




				def        lowercase__     (  self		:					Any,						lowerCamelCase		:					Dict,						lowerCamelCase		:					Dict						):

								'''simple docstring'''
								if self.config_name == "axb":
												return {"matthews_correlation": matthews_corrcoef(__a,						__a						)}
								elif self.config_name == "cb":
												return acc_and_fa(__a,						__a,						fa_avg='''macro'''						)
								elif self.config_name == "record":
												lowercase__     				=    [
												    {
												        "qas": [
												            {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]}
												            for ref in references
												        ]
												    }
												]
												lowercase__     				=    {pred["idx"]["query"]: pred["prediction_text"] for pred in predictions}
												return evaluate_record(__a,						__a						)[0]
								elif self.config_name == "multirc":
												return evaluate_multirc(__a,						__a						)
								elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
												return {"accuracy": simple_accuracy(__a,						__a						)}
								else:
												raise KeyError(
												    '''You should supply a configuration name selected in '''
												    '''[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]'''						)







from __future__ import annotations

from collections import deque
from collections.abc import Iterator
from dataclasses import dataclass
@dataclass
class    _UpperCAmelCase      :

				"""simple docstring"""




				lowercase__       							= 42
				lowercase__       							= 42



class    _UpperCAmelCase      :

				"""simple docstring"""




				def __init__(  self		:					str,						lowerCamelCase		:					int						):

								'''simple docstring'''
								lowercase__     				=    [[] for _ in range(lowerCamelCase						)]
								lowercase__     				=    size




				def __getitem__(  self		:					Optional[Any],						lowerCamelCase		:					int						):

								'''simple docstring'''
								return iter(self._graph[vertex]						)




				@property
				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								return self._size




				def        lowercase__     (  self		:					Union[str, Any],						lowerCamelCase		:					int,						lowerCamelCase		:					int,						lowerCamelCase		:					int						):

								'''simple docstring'''
								if weight not in (0, 1):
												raise ValueError('''Edge weight must be either 0 or 1.'''						)

								if to_vertex < 0 or to_vertex >= self.size:
												raise ValueError('''Vertex indexes must be in [0; size).'''						)

								self._graph[from_vertex].append(Edge(lowerCamelCase,						lowerCamelCase						)						)




				def        lowercase__     (  self		:					Optional[int],						lowerCamelCase		:					int,						lowerCamelCase		:					int						):

								'''simple docstring'''
								lowercase__     				=    deque([start_vertex]						)
								lowercase__     				=    [None] * self.size
								lowercase__     				=    0

								while queue:
												lowercase__     				=    queue.popleft()
												lowercase__     				=    distances[current_vertex]
												if current_distance is None:
																continue

												for edge in self[current_vertex]:
																lowercase__     				=    current_distance + edge.weight
																lowercase__     				=    distances[edge.destination_vertex]
																if (
																    isinstance(lowerCamelCase,						lowerCamelCase						)
																    and new_distance >= dest_vertex_distance
																):
																				continue
																lowercase__     				=    new_distance
																if edge.weight == 0:
																				queue.appendleft(edge.destination_vertex						)
																else:
																				queue.append(edge.destination_vertex						)

								if distances[finish_vertex] is None:
												raise ValueError('''No path from start_vertex to finish_vertex.'''						)

								return distances[finish_vertex]


if __name__ == "__main__":
							import doctest

							doctest.testmod()




