



import numpy as np
from scipy.spatial.distance import cdist
from sklearn.metrics import fa_score

import datasets


UpperCAmelCase_									=							"\\n    @inproceedings{kakwani2020indicnlpsuite,\n    title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}},\n    author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar},\n    year={2020},\n    booktitle={Findings of EMNLP},\n}\n"

UpperCAmelCase_									=							"\\n    IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide\n    variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te.\n"

UpperCAmelCase_									=							"\nCompute IndicGLUE evaluation metric associated to each IndicGLUE dataset.\nArgs:\n    predictions: list of predictions to score (as int64),\n        except for 'cvit-mkb-clsr' where each prediction is a vector (of float32).\n    references: list of ground truth labels corresponding to the predictions (as int64),\n        except for 'cvit-mkb-clsr' where each reference is a vector (of float32).\nReturns: depending on the IndicGLUE subset, one or several of:\n    \"accuracy\": Accuracy\n    \"f1\": F1 score\n    \"precision\": Precision@10\nExamples:\n\n    >>> indic_glue_metric = datasets.load_metric('indic_glue', 'wnli')  # 'wnli' or any of [\"copa\", \"sna\", \"csqa\", \"wstp\", \"inltkh\", \"bbca\", \"iitp-mr\", \"iitp-pr\", \"actsa-sc\", \"md\"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric('indic_glue', 'wiki-ner')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric('indic_glue', 'cvit-mkb-clsr')\n    >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'precision@10': 1.0}\n\n"
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Optional[Any]     ,				_SCREAMING_SNAKE_CASE   :	Optional[int]				)->Dict:
      return float((preds == labels).mean()				)
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Union[str, Any]     ,				_SCREAMING_SNAKE_CASE   :	Tuple				)->Dict:
      _lowerCAmelCase 			=  simple_accuracy(_SCREAMING_SNAKE_CASE     ,				_SCREAMING_SNAKE_CASE				)
      _lowerCAmelCase 			=  float(fa_score(y_true=_SCREAMING_SNAKE_CASE     ,				y_pred=_SCREAMING_SNAKE_CASE				)				)
      return {
          "accuracy": acc,
          "f1": fa,
      }
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Optional[int]     ,				_SCREAMING_SNAKE_CASE   :	str				)->Optional[int]:
      _lowerCAmelCase 			=  np.array(_SCREAMING_SNAKE_CASE				)
      _lowerCAmelCase 			=  np.array(_SCREAMING_SNAKE_CASE				)
      _lowerCAmelCase 			=  en_sentvecs.shape[0]

      # mean centering
      _lowerCAmelCase 			=  en_sentvecs - np.mean(_SCREAMING_SNAKE_CASE     ,				axis=0				)
      _lowerCAmelCase 			=  in_sentvecs - np.mean(_SCREAMING_SNAKE_CASE     ,				axis=0				)

      _lowerCAmelCase 			=  cdist(_SCREAMING_SNAKE_CASE     ,				_SCREAMING_SNAKE_CASE     ,				'''cosine'''				)
      _lowerCAmelCase 			=  np.array(range(_SCREAMING_SNAKE_CASE				)				)
      _lowerCAmelCase 			=  sim.argsort(axis=1				)[:, :1_0]
      _lowerCAmelCase 			=  np.any(preds == actual[:, None]     ,				axis=1				)
      return float(matches.mean()				)





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION       ,_KWARGS_DESCRIPTION				)
class 		UpperCAmelCase						(						datasets.Metric				):





   def 	__lowerCAmelCase							(	self			):
         if self.config_name not in [
             "wnli",
             "copa",
             "sna",
             "csqa",
             "wstp",
             "inltkh",
             "bbca",
             "cvit-mkb-clsr",
             "iitp-mr",
             "iitp-pr",
             "actsa-sc",
             "md",
             "wiki-ner",
         ]:
               raise KeyError(
                   '''You should supply a configuration name selected in '''
                   '''["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '''
                   '''"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '''
                   '''"wiki-ner"]'''			)
         return datasets.MetricInfo(
             description=_DESCRIPTION       ,     citation=_CITATION       ,     inputs_description=_KWARGS_DESCRIPTION       ,     features=datasets.Features(
                 {
                     '''predictions''': datasets.Value('''int64'''			)
                     if self.config_name != '''cvit-mkb-clsr'''
                     else datasets.Sequence(datasets.Value('''float32'''			)			),
                     '''references''': datasets.Value('''int64'''			)
                     if self.config_name != '''cvit-mkb-clsr'''
                     else datasets.Sequence(datasets.Value('''float32'''			)			),
                 }			)       ,     codebase_urls=[]       ,     reference_urls=[]       ,     format='''numpy''' if self.config_name != '''cvit-mkb-clsr''' else None       ,     )





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         if self.config_name == "cvit-mkb-clsr":
               return {"precision@10": precision_at_aa(_lowerCAmelCase       ,     _lowerCAmelCase			)}
         elif self.config_name in ["wiki-ner"]:
               return acc_and_fa(_lowerCAmelCase       ,     _lowerCAmelCase			)
         elif self.config_name in [
             "wnli",
             "copa",
             "sna",
             "csqa",
             "wstp",
             "inltkh",
             "bbca",
             "iitp-mr",
             "iitp-pr",
             "actsa-sc",
             "md",
         ]:
               return {"accuracy": simple_accuracy(_lowerCAmelCase       ,     _lowerCAmelCase			)}
         else:
               raise KeyError(
                   '''You should supply a configuration name selected in '''
                   '''["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '''
                   '''"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '''
                   '''"wiki-ner"]'''			)




from tempfile import TemporaryDirectory
from unittest import TestCase
from unittest.mock import MagicMock, patch

from transformers import AutoModel, TFAutoModel
from transformers.onnx import FeaturesManager
from transformers.testing_utils import SMALL_MODEL_IDENTIFIER, require_tf, require_torch





@require_torch
@require_tf
class 		UpperCAmelCase						(						snake_case_				):





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  SMALL_MODEL_IDENTIFIER
         _lowerCAmelCase 			=  '''pt'''
         _lowerCAmelCase 			=  '''tf'''





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  AutoModel.from_pretrained(self.test_model			)
         model_pt.save_pretrained(_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  TFAutoModel.from_pretrained(self.test_model       ,     from_pt=_lowerCAmelCase			)
         model_tf.save_pretrained(_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  '''mock_framework'''

         # Framework provided - return whatever the user provides
         _lowerCAmelCase 			=  FeaturesManager.determine_framework(self.test_model       ,     _lowerCAmelCase			)
         self.assertEqual(_lowerCAmelCase       ,     _lowerCAmelCase			)

         # Local checkpoint and framework provided - return provided framework
         # PyTorch checkpoint
         with TemporaryDirectory() as local_pt_ckpt:
               self._setup_pt_ckpt(_lowerCAmelCase			)
               _lowerCAmelCase 			=  FeaturesManager.determine_framework(_lowerCAmelCase       ,     _lowerCAmelCase			)
               self.assertEqual(_lowerCAmelCase       ,     _lowerCAmelCase			)

         # TensorFlow checkpoint
         with TemporaryDirectory() as local_tf_ckpt:
               self._setup_tf_ckpt(_lowerCAmelCase			)
               _lowerCAmelCase 			=  FeaturesManager.determine_framework(_lowerCAmelCase       ,     _lowerCAmelCase			)
               self.assertEqual(_lowerCAmelCase       ,     _lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         # PyTorch checkpoint
         with TemporaryDirectory() as local_pt_ckpt:
               self._setup_pt_ckpt(_lowerCAmelCase			)
               _lowerCAmelCase 			=  FeaturesManager.determine_framework(_lowerCAmelCase			)
               self.assertEqual(_lowerCAmelCase       ,     self.framework_pt			)

         # TensorFlow checkpoint
         with TemporaryDirectory() as local_tf_ckpt:
               self._setup_tf_ckpt(_lowerCAmelCase			)
               _lowerCAmelCase 			=  FeaturesManager.determine_framework(_lowerCAmelCase			)
               self.assertEqual(_lowerCAmelCase       ,     self.framework_tf			)

         # Invalid local checkpoint
         with TemporaryDirectory() as local_invalid_ckpt:
               with self.assertRaises(_lowerCAmelCase			):
                     _lowerCAmelCase 			=  FeaturesManager.determine_framework(_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  MagicMock(return_value=_lowerCAmelCase			)
         with patch('''transformers.onnx.features.is_tf_available'''       ,     _lowerCAmelCase			):
               _lowerCAmelCase 			=  FeaturesManager.determine_framework(self.test_model			)
               self.assertEqual(_lowerCAmelCase       ,     self.framework_pt			)

         # PyTorch not in environment -> use TensorFlow
         _lowerCAmelCase 			=  MagicMock(return_value=_lowerCAmelCase			)
         with patch('''transformers.onnx.features.is_torch_available'''       ,     _lowerCAmelCase			):
               _lowerCAmelCase 			=  FeaturesManager.determine_framework(self.test_model			)
               self.assertEqual(_lowerCAmelCase       ,     self.framework_tf			)

         # Both in environment -> use PyTorch
         _lowerCAmelCase 			=  MagicMock(return_value=_lowerCAmelCase			)
         _lowerCAmelCase 			=  MagicMock(return_value=_lowerCAmelCase			)
         with patch('''transformers.onnx.features.is_tf_available'''       ,     _lowerCAmelCase			), patch(
             '''transformers.onnx.features.is_torch_available'''       ,     _lowerCAmelCase			):
               _lowerCAmelCase 			=  FeaturesManager.determine_framework(self.test_model			)
               self.assertEqual(_lowerCAmelCase       ,     self.framework_pt			)

         # Both not in environment -> raise error
         _lowerCAmelCase 			=  MagicMock(return_value=_lowerCAmelCase			)
         _lowerCAmelCase 			=  MagicMock(return_value=_lowerCAmelCase			)
         with patch('''transformers.onnx.features.is_tf_available'''       ,     _lowerCAmelCase			), patch(
             '''transformers.onnx.features.is_torch_available'''       ,     _lowerCAmelCase			):
               with self.assertRaises(_lowerCAmelCase			):
                     _lowerCAmelCase 			=  FeaturesManager.determine_framework(self.test_model			)