






'''simple docstring'''







from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__lowerCAmelCase       		=      '\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n'

__lowerCAmelCase       		=      '\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n'

__lowerCAmelCase       		=      '\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for \'record\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'prediction_text\': the predicted answer text\n        - for \'multirc\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question-answer pair as specified by the dataset\n            - \'prediction\': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for \'record\': list of question-answers dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'answers\': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for \'record\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1\': F1 score\n    - for \'multirc\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1_m\': Per-question macro-F1 score\n        - \'f1_a\': Average F1 score over all answers\n    - for \'axb\':\n        \'matthews_correlation\': Matthew Correlation\n    - for \'cb\':\n        - \'accuracy\': Accuracy\n        - \'f1\': F1 score\n    - for all others:\n        - \'accuracy\': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')\n    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]\n    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')\n    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'





def 		_UpperCAmelCase   (							__A							:   Any						,		__A							:   str      ):
				return float((preds == labels).mean()      )





def 		_UpperCAmelCase   (							__A							:   List[Any]						,		__A							:   Optional[Any]						,		__A							:   str="binary"      ):
				a_		:  Dict											=      simple_accuracy(__A						,		__A      )
				a_		:  Tuple											=      float(fa_score(y_true=__A						,		y_pred=__A						,		average=__A      )      )
				return {
				    "accuracy": acc,
				    "f1": fa,
				}





def 		_UpperCAmelCase   (							__A							:   Union[str, Any]						,		__A							:   Dict      ):
				a_		:  Dict											=      {}
				for id_pred, label in zip(__A						,		__A      ):
								a_		:  int											=      f'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
								a_		:  int											=      id_pred['''prediction''']
								if question_id in question_map:
												question_map[question_id].append((pred, label)      )
								else:
												a_		:  List[Any]											=      [(pred, label)]
				a_				,			a_		:  Dict											=      [], []
				for question, preds_labels in question_map.items():
								a_				,			a_		:  List[Any]											=      zip(*__A      )
								a_		:  List[str]											=      fa_score(y_true=__A						,		y_pred=__A						,		average='''macro'''      )
								fas.append(__A      )
								a_		:  Any											=      int(sum(pred == label for pred, label in preds_labels      ) == len(__A      )      )
								ems.append(__A      )
				a_		:  str											=      float(sum(__A      ) / len(__A      )      )
				a_		:  int											=      sum(__A      ) / len(__A      )
				a_		:  Dict											=      float(fa_score(y_true=__A						,		y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]      )      )
				return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,   _KWARGS_DESCRIPTION       )
class 	SCREAMING_SNAKE_CASE					(					datasets.Metric       ):






			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict					)			->			Optional[Any]:
							if self.config_name not in [
							    "boolq",
							    "cb",
							    "copa",
							    "multirc",
							    "record",
							    "rte",
							    "wic",
							    "wsc",
							    "wsc.fixed",
							    "axb",
							    "axg",
							]:
											raise KeyError(
											    '''You should supply a configuration name selected in '''
											    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''					)
							return datasets.MetricInfo(
							    description=_DESCRIPTION  ,						citation=_CITATION  ,						inputs_description=_KWARGS_DESCRIPTION  ,						features=datasets.Features(self._get_feature_types()					)  ,						codebase_urls=[]  ,						reference_urls=[]  ,						format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None  ,						)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			str:
							if self.config_name == "record":
											return {
											    "predictions": {
											        "idx": {
											            "passage": datasets.Value('''int64'''					),
											            "query": datasets.Value('''int64'''					),
											        },
											        "prediction_text": datasets.Value('''string'''					),
											    },
											    "references": {
											        "idx": {
											            "passage": datasets.Value('''int64'''					),
											            "query": datasets.Value('''int64'''					),
											        },
											        "answers": datasets.Sequence(datasets.Value('''string'''					)					),
											    },
											}
							elif self.config_name == "multirc":
											return {
											    "predictions": {
											        "idx": {
											            "answer": datasets.Value('''int64'''					),
											            "paragraph": datasets.Value('''int64'''					),
											            "question": datasets.Value('''int64'''					),
											        },
											        "prediction": datasets.Value('''int64'''					),
											    },
											    "references": datasets.Value('''int64'''					),
											}
							else:
											return {
											    "predictions": datasets.Value('''int64'''					),
											    "references": datasets.Value('''int64'''					),
											}







			def 			SCREAMING_SNAKE_CASE      ( self      :			Any  ,						__SCREAMING_SNAKE_CASE      :			int  ,						__SCREAMING_SNAKE_CASE      :			Dict					)			->			str:
							if self.config_name == "axb":
											return {"matthews_correlation": matthews_corrcoef(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)}
							elif self.config_name == "cb":
											return acc_and_fa(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE  ,						fa_avg='''macro'''					)
							elif self.config_name == "record":
											a_		:  Optional[int]											=      [
											    {
											        '''qas''': [
											            {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
											            for ref in references
											        ]
											    }
											]
											a_		:  List[Any]											=      {pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
											return evaluate_record(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)[0]
							elif self.config_name == "multirc":
											return evaluate_multirc(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)
							elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
											return {"accuracy": simple_accuracy(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)}
							else:
											raise KeyError(
											    '''You should supply a configuration name selected in '''
											    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''					)












'''simple docstring'''







from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional

from packaging import version


if TYPE_CHECKING:
		from ... import PreTrainedTokenizer, TensorType

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import is_torch_available, logging


__lowerCAmelCase       		=      logging.get_logger(__name__)

__lowerCAmelCase       		=      {
    'bigscience/bloom': 'https://huggingface.co/bigscience/bloom/resolve/main/config.json',
    'bigscience/bloom-560m': 'https://huggingface.co/bigscience/bloom-560m/blob/main/config.json',
    'bigscience/bloom-1b1': 'https://huggingface.co/bigscience/bloom-1b1/blob/main/config.json',
    'bigscience/bloom-1b7': 'https://huggingface.co/bigscience/bloom-1b7/blob/main/config.json',
    'bigscience/bloom-3b': 'https://huggingface.co/bigscience/bloom-3b/blob/main/config.json',
    'bigscience/bloom-7b1': 'https://huggingface.co/bigscience/bloom-7b1/blob/main/config.json',
}





class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				"bloom"
			snake_case__  =				["past_key_values"]
			snake_case__  =				{
			    "num_hidden_layers": "n_layer",
			    "num_attention_heads": "n_head",
			}






			def __init__( self      :			Union[str, Any]  ,						__SCREAMING_SNAKE_CASE      :			int=25_0880  ,						__SCREAMING_SNAKE_CASE      :			Dict=64  ,						__SCREAMING_SNAKE_CASE      :			Tuple=2  ,						__SCREAMING_SNAKE_CASE      :			int=8  ,						__SCREAMING_SNAKE_CASE      :			Any=1e-5  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=0.02  ,						__SCREAMING_SNAKE_CASE      :			Union[str, Any]=True  ,						__SCREAMING_SNAKE_CASE      :			int=1  ,						__SCREAMING_SNAKE_CASE      :			Any=2  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=False  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=0.0  ,						__SCREAMING_SNAKE_CASE      :			str=0.0  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=1  ,						__SCREAMING_SNAKE_CASE      :			List[str]=False  ,						**__SCREAMING_SNAKE_CASE      :			str  ,						)			->			Any:
							a_		:  Optional[int]											=      vocab_size
							# Backward compatibility with n_embed kwarg
							a_		:  Any											=      kwargs.pop('''n_embed'''  ,						__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[int]											=      hidden_size if n_embed is None else n_embed
							a_		:  int											=      n_layer
							a_		:  str											=      n_head
							a_		:  Optional[int]											=      layer_norm_epsilon
							a_		:  Dict											=      initializer_range
							a_		:  List[str]											=      use_cache
							a_		:  Dict											=      pretraining_tp
							a_		:  Optional[Any]											=      apply_residual_connection_post_layernorm
							a_		:  Optional[Any]											=      hidden_dropout
							a_		:  List[str]											=      attention_dropout

							a_		:  Dict											=      bos_token_id
							a_		:  Optional[int]											=      eos_token_id
							a_		:  Any											=      slow_but_exact

							super().__init__(bos_token_id=__SCREAMING_SNAKE_CASE  ,						eos_token_id=__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)





class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				version.parse("1.12"       )






			def __init__( self      :			Tuple  ,						__SCREAMING_SNAKE_CASE      :			PretrainedConfig  ,						__SCREAMING_SNAKE_CASE      :			str = "default"  ,						__SCREAMING_SNAKE_CASE      :			List[PatchingSpec] = None  ,						__SCREAMING_SNAKE_CASE      :			bool = False  ,						)			->			Optional[Any]:
							super().__init__(__SCREAMING_SNAKE_CASE  ,						task=__SCREAMING_SNAKE_CASE  ,						patching_specs=__SCREAMING_SNAKE_CASE  ,						use_past=__SCREAMING_SNAKE_CASE					)
							if not getattr(self._config  ,						'''pad_token_id'''  ,						__SCREAMING_SNAKE_CASE					):
											# TODO: how to do that better?
											a_		:  Tuple											=      0






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Mapping[str, Mapping[int, str]]:
							a_		:  Optional[Any]											=      OrderedDict({'''input_ids''': {0: '''batch''', 1: '''sequence'''}}					)
							if self.use_past:
											# BLOOM stores values on dynamic axis 2. For more details see: https://github.com/huggingface/transformers/pull/18344
											self.fill_with_past_key_values_(__SCREAMING_SNAKE_CASE  ,						direction='''inputs'''  ,						inverted_values_shape=__SCREAMING_SNAKE_CASE					)
											a_		:  Union[str, Any]											=      {0: '''batch''', 1: '''past_sequence + sequence'''}
							else:
											a_		:  Union[str, Any]											=      {0: '''batch''', 1: '''sequence'''}

							return common_inputs






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			int:
							return self._config.n_layer






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			int:
							return self._config.n_head






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			float:
							return 1e-3






			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict  ,						__SCREAMING_SNAKE_CASE      :			"PreTrainedTokenizer"  ,						__SCREAMING_SNAKE_CASE      :			int = -1  ,						__SCREAMING_SNAKE_CASE      :			int = -1  ,						__SCREAMING_SNAKE_CASE      :			bool = False  ,						__SCREAMING_SNAKE_CASE      :			Optional["TensorType"] = None  ,						)			->			Mapping[str, Any]:
							a_		:  Dict											=      super(__SCREAMING_SNAKE_CASE  ,						self					).generate_dummy_inputs(
							    __SCREAMING_SNAKE_CASE  ,						batch_size=__SCREAMING_SNAKE_CASE  ,						seq_length=__SCREAMING_SNAKE_CASE  ,						is_pair=__SCREAMING_SNAKE_CASE  ,						framework=__SCREAMING_SNAKE_CASE					)

							# We need to order the input in the way they appears in the forward()
							a_		:  Union[str, Any]											=      OrderedDict({'''input_ids''': common_inputs['''input_ids''']}					)

							# Need to add the past_keys
							if self.use_past:
											if not is_torch_available():
															raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''					)
											else:
															import torch

															a_				,			a_		:  Any											=      common_inputs['''input_ids'''].shape
															# Not using the same length for past_key_values
															a_		:  str											=      seqlen + 2
															a_		:  Any											=      self._config.hidden_size // self.num_attention_heads
															a_		:  Optional[int]											=      (
															    batch * self.num_attention_heads,
															    head_dim,
															    past_key_values_length,
															)
															a_		:  Any											=      (
															    batch * self.num_attention_heads,
															    past_key_values_length,
															    head_dim,
															)
															a_		:  List[str]											=      [
															    (torch.zeros(__SCREAMING_SNAKE_CASE					), torch.zeros(__SCREAMING_SNAKE_CASE					)) for _ in range(self.num_layers					)
															]

							a_		:  Union[str, Any]											=      common_inputs['''attention_mask''']
							if self.use_past:
											a_		:  Optional[int]											=      ordered_inputs['''attention_mask'''].dtype
											a_		:  List[Any]											=      torch.cat(
											    [ordered_inputs['''attention_mask'''], torch.ones(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE  ,						dtype=__SCREAMING_SNAKE_CASE					)]  ,						dim=1					)

							return ordered_inputs







			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[int]					)			->			int:
							return 13




