"""simple docstring"""




from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__lowerCAmelCase    :							List[Any]													=						"\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

__lowerCAmelCase    :							List[str]													=						"\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

__lowerCAmelCase    :							Tuple													=						"\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"







def 							_UpperCAmelCase		(       lowerCamelCase__			,  lowerCamelCase__     ):





		"""simple docstring"""




		return float((preds == labels).mean()     )







def 							_UpperCAmelCase		(       lowerCamelCase__			,  lowerCamelCase__			,  lowerCamelCase__="binary"     ):





		"""simple docstring"""




		lowerCAmelCase__        =   simple_accuracy(lowerCamelCase__			,  lowerCamelCase__     )
		lowerCAmelCase__        =   float(fa_score(y_true=lowerCamelCase__			,  y_pred=lowerCamelCase__			,  average=lowerCamelCase__     )     )
		return {
		    "accuracy": acc,
		    "f1": fa,
		}







def 							_UpperCAmelCase		(       lowerCamelCase__			,  lowerCamelCase__     ):





		"""simple docstring"""




		lowerCAmelCase__        =   {}
		for id_pred, label in zip(lowerCamelCase__			,  lowerCamelCase__     ):
				lowerCAmelCase__        =   f"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
				lowerCAmelCase__        =   id_pred["""prediction"""]
				if question_id in question_map:
						question_map[question_id].append((pred, label)     )
				else:
						lowerCAmelCase__        =   [(pred, label)]
		lowerCAmelCase__   ,					lowerCAmelCase__        =   [], []
		for question, preds_labels in question_map.items():
				lowerCAmelCase__   ,					lowerCAmelCase__        =   zip(*lowerCamelCase__     )
				lowerCAmelCase__        =   fa_score(y_true=lowerCamelCase__			,  y_pred=lowerCamelCase__			,  average="""macro"""     )
				fas.append(lowerCamelCase__     )
				lowerCAmelCase__        =   int(sum(pred == label for pred, label in preds_labels     ) == len(lowerCamelCase__     )     )
				ems.append(lowerCamelCase__     )
		lowerCAmelCase__        =   float(sum(lowerCamelCase__     ) / len(lowerCamelCase__     )     )
		lowerCAmelCase__        =   sum(lowerCamelCase__     ) / len(lowerCamelCase__     )
		lowerCAmelCase__        =   float(fa_score(y_true=lowerCamelCase__			,  y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]     )     )
		return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,		_KWARGS_DESCRIPTION			)
class       a_      (     datasets.Metric			):




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Any     ):
							if self.config_name not in [
							    "boolq",
							    "cb",
							    "copa",
							    "multirc",
							    "record",
							    "rte",
							    "wic",
							    "wsc",
							    "wsc.fixed",
							    "axb",
							    "axg",
							]:
									raise KeyError(
									    """You should supply a configuration name selected in """
									    """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""     )
							return datasets.MetricInfo(
							    description=_DESCRIPTION  ,						citation=_CITATION  ,						inputs_description=_KWARGS_DESCRIPTION  ,						features=datasets.Features(self._get_feature_types()     )  ,						codebase_urls=[]  ,						reference_urls=[]  ,						format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None  ,						)




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]     ):
							if self.config_name == "record":
									return {
									    "predictions": {
									        "idx": {
									            "passage": datasets.Value("""int64"""     ),
									            "query": datasets.Value("""int64"""     ),
									        },
									        "prediction_text": datasets.Value("""string"""     ),
									    },
									    "references": {
									        "idx": {
									            "passage": datasets.Value("""int64"""     ),
									            "query": datasets.Value("""int64"""     ),
									        },
									        "answers": datasets.Sequence(datasets.Value("""string"""     )     ),
									    },
									}
							elif self.config_name == "multirc":
									return {
									    "predictions": {
									        "idx": {
									            "answer": datasets.Value("""int64"""     ),
									            "paragraph": datasets.Value("""int64"""     ),
									            "question": datasets.Value("""int64"""     ),
									        },
									        "prediction": datasets.Value("""int64"""     ),
									    },
									    "references": datasets.Value("""int64"""     ),
									}
							else:
									return {
									    "predictions": datasets.Value("""int64"""     ),
									    "references": datasets.Value("""int64"""     ),
									}




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Dict  ,						snake_case__						:     int  ,						snake_case__						:     Union[str, Any]     ):
							if self.config_name == "axb":
									return {"matthews_correlation": matthews_corrcoef(snake_case__  ,						snake_case__     )}
							elif self.config_name == "cb":
									return acc_and_fa(snake_case__  ,						snake_case__  ,						fa_avg="""macro"""     )
							elif self.config_name == "record":
									lowerCAmelCase__        =   [
									    {
									        """qas""": [
									            {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
									            for ref in references
									        ]
									    }
									]
									lowerCAmelCase__        =   {pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
									return evaluate_record(snake_case__  ,						snake_case__     )[0]
							elif self.config_name == "multirc":
									return evaluate_multirc(snake_case__  ,						snake_case__     )
							elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
									return {"accuracy": simple_accuracy(snake_case__  ,						snake_case__     )}
							else:
									raise KeyError(
									    """You should supply a configuration name selected in """
									    """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""     )





"""simple docstring"""




from __future__ import annotations

import inspect
import unittest

import numpy as np

from transformers import DeiTConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import (
			    TFDeiTForImageClassification,
			    TFDeiTForImageClassificationWithTeacher,
			    TFDeiTForMaskedImageModeling,
			    TFDeiTModel,
			)
			from transformers.models.deit.modeling_tf_deit import TF_DEIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
			from PIL import Image

			from transformers import DeiTImageProcessor

class       a_      :




					def __init__(	self						:     Optional[int]  ,						snake_case__						:     Optional[Any]  ,						snake_case__						:     Any=13  ,						snake_case__						:     int=30  ,						snake_case__						:     int=2  ,						snake_case__						:     Union[str, Any]=3  ,						snake_case__						:     Dict=True  ,						snake_case__						:     Optional[int]=True  ,						snake_case__						:     List[Any]=32  ,						snake_case__						:     List[str]=2  ,						snake_case__						:     Optional[Any]=4  ,						snake_case__						:     Optional[int]=37  ,						snake_case__						:     Tuple="gelu"  ,						snake_case__						:     str=0.1  ,						snake_case__						:     Any=0.1  ,						snake_case__						:     int=10  ,						snake_case__						:     Dict=0.02  ,						snake_case__						:     Union[str, Any]=3  ,						snake_case__						:     str=None  ,						snake_case__						:     List[Any]=2  ,						):
							lowerCAmelCase__        =   parent
							lowerCAmelCase__        =   batch_size
							lowerCAmelCase__        =   image_size
							lowerCAmelCase__        =   patch_size
							lowerCAmelCase__        =   num_channels
							lowerCAmelCase__        =   is_training
							lowerCAmelCase__        =   use_labels
							lowerCAmelCase__        =   hidden_size
							lowerCAmelCase__        =   num_hidden_layers
							lowerCAmelCase__        =   num_attention_heads
							lowerCAmelCase__        =   intermediate_size
							lowerCAmelCase__        =   hidden_act
							lowerCAmelCase__        =   hidden_dropout_prob
							lowerCAmelCase__        =   attention_probs_dropout_prob
							lowerCAmelCase__        =   type_sequence_label_size
							lowerCAmelCase__        =   initializer_range
							lowerCAmelCase__        =   scope
							lowerCAmelCase__        =   encoder_stride

							# in DeiT, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distilation tokens)
							lowerCAmelCase__        =   (image_size // patch_size) ** 2
							lowerCAmelCase__        =   num_patches + 2




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Any     ):
							lowerCAmelCase__        =   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]     )

							lowerCAmelCase__        =   None
							if self.use_labels:
									lowerCAmelCase__        =   ids_tensor([self.batch_size]  ,						self.type_sequence_label_size     )

							lowerCAmelCase__        =   self.get_config()

							return config, pixel_values, labels




					def 						_SCREAMING_SNAKE_CASE						(	self						:     List[Any]     ):
							return DeiTConfig(
							    image_size=self.image_size  ,						patch_size=self.patch_size  ,						num_channels=self.num_channels  ,						hidden_size=self.hidden_size  ,						num_hidden_layers=self.num_hidden_layers  ,						num_attention_heads=self.num_attention_heads  ,						intermediate_size=self.intermediate_size  ,						hidden_act=self.hidden_act  ,						hidden_dropout_prob=self.hidden_dropout_prob  ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,						is_decoder=snake_case__  ,						initializer_range=self.initializer_range  ,						encoder_stride=self.encoder_stride  ,						)




					def 						_SCREAMING_SNAKE_CASE						(	self						:     int  ,						snake_case__						:     int  ,						snake_case__						:     Optional[Any]  ,						snake_case__						:     List[str]     ):
							lowerCAmelCase__        =   TFDeiTModel(config=snake_case__     )
							lowerCAmelCase__        =   model(snake_case__     )
							self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, self.seq_length, self.hidden_size)     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Tuple  ,						snake_case__						:     Dict  ,						snake_case__						:     Dict  ,						snake_case__						:     Dict     ):
							lowerCAmelCase__        =   TFDeiTForMaskedImageModeling(config=snake_case__     )
							lowerCAmelCase__        =   model(snake_case__     )
							self.parent.assertEqual(
							    result.reconstruction.shape  ,						(self.batch_size, self.num_channels, self.image_size, self.image_size)     )

							# test greyscale images
							lowerCAmelCase__        =   1
							lowerCAmelCase__        =   TFDeiTForMaskedImageModeling(snake_case__     )

							lowerCAmelCase__        =   floats_tensor([self.batch_size, 1, self.image_size, self.image_size]     )
							lowerCAmelCase__        =   model(snake_case__     )
							self.parent.assertEqual(result.reconstruction.shape  ,						(self.batch_size, 1, self.image_size, self.image_size)     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Optional[Any]  ,						snake_case__						:     Union[str, Any]  ,						snake_case__						:     Any  ,						snake_case__						:     Tuple     ):
							lowerCAmelCase__        =   self.type_sequence_label_size
							lowerCAmelCase__        =   TFDeiTForImageClassification(snake_case__     )
							lowerCAmelCase__        =   model(snake_case__  ,						labels=snake_case__     )
							self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.type_sequence_label_size)     )

							# test greyscale images
							lowerCAmelCase__        =   1
							lowerCAmelCase__        =   TFDeiTForImageClassification(snake_case__     )

							lowerCAmelCase__        =   floats_tensor([self.batch_size, 1, self.image_size, self.image_size]     )
							lowerCAmelCase__        =   model(snake_case__  ,						labels=snake_case__     )
							self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.type_sequence_label_size)     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Optional[int]     ):
							lowerCAmelCase__        =   self.prepare_config_and_inputs()
							lowerCAmelCase__   ,					lowerCAmelCase__   ,					lowerCAmelCase__        =   config_and_inputs
							lowerCAmelCase__        =   {"""pixel_values""": pixel_values}
							return config, inputs_dict

@require_tf
class       a_      (     __UpperCamelCase ,		__UpperCamelCase ,		unittest.TestCase			):
					UpperCamelCase_ :	Optional[Any]			=    (
					    (
					        TFDeiTModel,
					        TFDeiTForImageClassification,
					        TFDeiTForImageClassificationWithTeacher,
					        TFDeiTForMaskedImageModeling,
					    )
					    if is_tf_available()
					    else ()
					)
					UpperCamelCase_ :	Any			=    (
					    {
					        "feature-extraction": TFDeiTModel,
					        "image-classification": (TFDeiTForImageClassification, TFDeiTForImageClassificationWithTeacher),
					    }
					    if is_tf_available()
					    else {}
					)

					UpperCamelCase_ :	Optional[Any]			=    False
					UpperCamelCase_ :	Optional[Any]			=    False
					UpperCamelCase_ :	Optional[int]			=    False
					UpperCamelCase_ :	int			=    False




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]     ):
							lowerCAmelCase__        =   TFDeiTModelTester(self     )
							lowerCAmelCase__        =   ConfigTester(self  ,						config_class=snake_case__  ,						has_text_modality=snake_case__  ,						hidden_size=37     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]     ):
							self.config_tester.run_common_tests()




					@unittest.skip(reason="""DeiT does not use inputs_embeds"""     )
					def 						_SCREAMING_SNAKE_CASE						(	self						:     Any     ):
							pass




					def 						_SCREAMING_SNAKE_CASE						(	self						:     str     ):
							lowerCAmelCase__   ,					lowerCAmelCase__        =   self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
									lowerCAmelCase__        =   model_class(snake_case__     )
									self.assertIsInstance(model.get_input_embeddings()  ,						(tf.keras.layers.Layer)     )
									lowerCAmelCase__        =   model.get_output_embeddings()
									self.assertTrue(x is None or isinstance(snake_case__  ,						tf.keras.layers.Dense     )     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Dict     ):
							lowerCAmelCase__   ,					lowerCAmelCase__        =   self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
									lowerCAmelCase__        =   model_class(snake_case__     )
									lowerCAmelCase__        =   inspect.signature(model.call     )
									# signature.parameters is an OrderedDict => so arg_names order is deterministic
									lowerCAmelCase__        =   [*signature.parameters.keys()]

									lowerCAmelCase__        =   ["""pixel_values"""]
									self.assertListEqual(arg_names[:1]  ,						snake_case__     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Tuple     ):
							lowerCAmelCase__        =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*snake_case__     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     int     ):
							lowerCAmelCase__        =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_image_modeling(*snake_case__     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     int     ):
							lowerCAmelCase__        =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_image_classification(*snake_case__     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]  ,						snake_case__						:     Union[str, Any]  ,						snake_case__						:     int  ,						snake_case__						:     Union[str, Any]=False     ):
							lowerCAmelCase__        =   super()._prepare_for_class(snake_case__  ,						snake_case__  ,						return_labels=snake_case__     )

							if return_labels:
									if "labels" in inputs_dict and "labels" not in inspect.signature(model_class.call     ).parameters:
											del inputs_dict["labels"]

							return inputs_dict




					@slow
					def 						_SCREAMING_SNAKE_CASE						(	self						:     Any     ):
							for model_name in TF_DEIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowerCAmelCase__        =   TFDeiTModel.from_pretrained(snake_case__     )
									self.assertIsNotNone(snake_case__     )







def 							_UpperCAmelCase		(       ):





		"""simple docstring"""




		lowerCAmelCase__        =   Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""     )
		return image






@require_tf
@require_vision
class       a_      (     unittest.TestCase			):




					@cached_property
					def 						_SCREAMING_SNAKE_CASE						(	self						:     Any     ):
							return (
							    DeiTImageProcessor.from_pretrained("""facebook/deit-base-distilled-patch16-224"""     )
							    if is_vision_available()
							    else None
							)




					@slow
					def 						_SCREAMING_SNAKE_CASE						(	self						:     List[Any]     ):
							lowerCAmelCase__        =   TFDeiTForImageClassificationWithTeacher.from_pretrained("""facebook/deit-base-distilled-patch16-224"""     )

							lowerCAmelCase__        =   self.default_image_processor
							lowerCAmelCase__        =   prepare_img()
							lowerCAmelCase__        =   image_processor(images=snake_case__  ,						return_tensors="""tf"""     )

							# forward pass
							lowerCAmelCase__        =   model(**snake_case__     )

							# verify the logits
							lowerCAmelCase__        =   tf.TensorShape((1, 1000)     )
							self.assertEqual(outputs.logits.shape  ,						snake_case__     )

							lowerCAmelCase__        =   tf.constant([-1.0266, 0.1912, -1.2861]     )

							self.assertTrue(np.allclose(outputs.logits[0, :3]  ,						snake_case__  ,						atol=1E-4     )     )




