





'''simple docstring'''







from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets


snake_case_     :       List[str]						=  "\\n@inproceedings{wang2019glue,\n  title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding},\n  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},\n  note={In the Proceedings of ICLR.},\n  year={2019}\n}\n"

snake_case_     :       Optional[Any]						=  "\\nGLUE, the General Language Understanding Evaluation benchmark\n(https://gluebenchmark.com/) is a collection of resources for training,\nevaluating, and analyzing natural language understanding systems.\n"

snake_case_     :       List[str]						=  "\nCompute GLUE evaluation metric associated to each GLUE dataset.\nArgs:\n    predictions: list of predictions to score.\n        Each translation should be tokenized into a list of tokens.\n    references: list of lists of references for each translation.\n        Each reference should be tokenized into a list of tokens.\nReturns: depending on the GLUE subset, one or several of:\n    \"accuracy\": Accuracy\n    \"f1\": F1 score\n    \"pearson\": Pearson Correlation\n    \"spearmanr\": Spearman Correlation\n    \"matthews_correlation\": Matthew Correlation\nExamples:\n\n    >>> glue_metric = datasets.load_metric('glue', 'sst2')  # 'sst2' or any of [\"mnli\", \"mnli_mismatched\", \"mnli_matched\", \"qnli\", \"rte\", \"wnli\", \"hans\"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> glue_metric = datasets.load_metric('glue', 'mrpc')  # 'mrpc' or 'qqp'\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> glue_metric = datasets.load_metric('glue', 'stsb')\n    >>> references = [0., 1., 2., 3., 4., 5.]\n    >>> predictions = [0., 1., 2., 3., 4., 5.]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print({\"pearson\": round(results[\"pearson\"], 2), \"spearmanr\": round(results[\"spearmanr\"], 2)})\n    {'pearson': 1.0, 'spearmanr': 1.0}\n\n    >>> glue_metric = datasets.load_metric('glue', 'cola')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			Dict,			SCREAMING_SNAKE_CASE__      :			Optional[int]   ) ->	Optional[Any]:
      return float((preds == labels).mean()   )





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			Union[str, Any],			SCREAMING_SNAKE_CASE__      :			Dict   ) ->	Optional[int]:
      UpperCAmelCase_			:      Union[str, Any]     						=  simple_accuracy(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )
      UpperCAmelCase_			:      Union[str, Any]     						=  float(fa_score(y_true=SCREAMING_SNAKE_CASE__,			y_pred=SCREAMING_SNAKE_CASE__   )   )
      return {
          "accuracy": acc,
          "f1": fa,
      }





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			List[str],			SCREAMING_SNAKE_CASE__      :			Optional[int]   ) ->	Tuple:
      UpperCAmelCase_			:      Optional[Any]     						=  float(pearsonr(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )[0]   )
      UpperCAmelCase_			:      Any     						=  float(spearmanr(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )[0]   )
      return {
          "pearson": pearson_corr,
          "spearmanr": spearman_corr,
      }


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION       ,		_KWARGS_DESCRIPTION     )
class 					__a      (datasets.Metric     ):

       def     UpperCAmelCase__	(							self							:     Optional[int]   )					->       int:







             """simple docstring"""
             if self.config_name not in [
                 "sst2",
                 "mnli",
                 "mnli_mismatched",
                 "mnli_matched",
                 "cola",
                 "stsb",
                 "mrpc",
                 "qqp",
                 "qnli",
                 "rte",
                 "wnli",
                 "hans",
             ]:
                   raise KeyError(
                       '''You should supply a configuration name selected in '''
                       '''["sst2", "mnli", "mnli_mismatched", "mnli_matched", '''
                       '''"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'''   )
             return datasets.MetricInfo(
                 description=_DESCRIPTION       ,     citation=_CITATION       ,     inputs_description=_KWARGS_DESCRIPTION       ,     features=datasets.Features(
                     {
                         '''predictions''': datasets.Value('''int64''' if self.config_name != '''stsb''' else '''float32'''   ),
                         '''references''': datasets.Value('''int64''' if self.config_name != '''stsb''' else '''float32'''   ),
                     }   )       ,     codebase_urls=[]       ,     reference_urls=[]       ,     format='''numpy'''       ,     )



       def     UpperCAmelCase__	(							self							:     Union[str, Any]       ,     __magic_name__							:     Union[str, Any]       ,     __magic_name__							:     Optional[Any]   )					->       Any:







             """simple docstring"""
             if self.config_name == "cola":
                   return {"matthews_correlation": matthews_corrcoef(__magic_name__       ,     __magic_name__   )}
             elif self.config_name == "stsb":
                   return pearson_and_spearman(__magic_name__       ,     __magic_name__   )
             elif self.config_name in ["mrpc", "qqp"]:
                   return acc_and_fa(__magic_name__       ,     __magic_name__   )
             elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
                   return {"accuracy": simple_accuracy(__magic_name__       ,     __magic_name__   )}
             else:
                   raise KeyError(
                       '''You should supply a configuration name selected in '''
                       '''["sst2", "mnli", "mnli_mismatched", "mnli_matched", '''
                       '''"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'''   )












'''simple docstring'''







import unittest

from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    LiltForQuestionAnswering,
					    LiltForSequenceClassification,
					    LiltForTokenClassification,
					    LiltModel,
					)
					from transformers.models.lilt.modeling_lilt import LILT_PRETRAINED_MODEL_ARCHIVE_LIST


class 					__a      :

							def __init__(							self							:     int       ,     __magic_name__							:     Optional[Any]       ,     __magic_name__							:     Any=13       ,     __magic_name__							:     Any=7       ,     __magic_name__							:     Union[str, Any]=True       ,     __magic_name__							:     Union[str, Any]=True       ,     __magic_name__							:     str=True       ,     __magic_name__							:     Optional[int]=True       ,     __magic_name__							:     List[Any]=99       ,     __magic_name__							:     int=24       ,     __magic_name__							:     Optional[int]=2       ,     __magic_name__							:     Tuple=6       ,     __magic_name__							:     Union[str, Any]=37       ,     __magic_name__							:     Optional[Any]="gelu"       ,     __magic_name__							:     Any=0.1       ,     __magic_name__							:     str=0.1       ,     __magic_name__							:     Tuple=5_12       ,     __magic_name__							:     Union[str, Any]=16       ,     __magic_name__							:     Tuple=2       ,     __magic_name__							:     Tuple=0.0_2       ,     __magic_name__							:     Optional[Any]=3       ,     __magic_name__							:     Optional[int]=None       ,     __magic_name__							:     Any=10_00       ,     )					->       str:







													"""simple docstring"""
													UpperCAmelCase_			:      Tuple     						=  parent
													UpperCAmelCase_			:      Optional[int]     						=  batch_size
													UpperCAmelCase_			:      List[str]     						=  seq_length
													UpperCAmelCase_			:      Dict     						=  is_training
													UpperCAmelCase_			:      List[str]     						=  use_input_mask
													UpperCAmelCase_			:      Any     						=  use_token_type_ids
													UpperCAmelCase_			:      Any     						=  use_labels
													UpperCAmelCase_			:      Any     						=  vocab_size
													UpperCAmelCase_			:      Dict     						=  hidden_size
													UpperCAmelCase_			:      Tuple     						=  num_hidden_layers
													UpperCAmelCase_			:      Tuple     						=  num_attention_heads
													UpperCAmelCase_			:      int     						=  intermediate_size
													UpperCAmelCase_			:      Union[str, Any]     						=  hidden_act
													UpperCAmelCase_			:      Optional[int]     						=  hidden_dropout_prob
													UpperCAmelCase_			:      Optional[Any]     						=  attention_probs_dropout_prob
													UpperCAmelCase_			:      Union[str, Any]     						=  max_position_embeddings
													UpperCAmelCase_			:      int     						=  type_vocab_size
													UpperCAmelCase_			:      List[Any]     						=  type_sequence_label_size
													UpperCAmelCase_			:      int     						=  initializer_range
													UpperCAmelCase_			:      Dict     						=  num_labels
													UpperCAmelCase_			:      List[str]     						=  scope
													UpperCAmelCase_			:      List[str]     						=  range_bbox

							def     UpperCAmelCase__	(							self							:     Optional[int]   )					->       List[Any]:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[int]     						=  ids_tensor([self.batch_size, self.seq_length]       ,     self.vocab_size   )

													UpperCAmelCase_			:      List[str]     						=  ids_tensor([self.batch_size, self.seq_length, 4]       ,     self.range_bbox   )
													# Ensure that bbox is legal
													for i in range(bbox.shape[0]   ):
																			for j in range(bbox.shape[1]   ):
																									if bbox[i, j, 3] < bbox[i, j, 1]:
																															UpperCAmelCase_			:      List[str]     						=  bbox[i, j, 3]
																															UpperCAmelCase_			:      Dict     						=  bbox[i, j, 1]
																															UpperCAmelCase_			:      Optional[Any]     						=  t
																									if bbox[i, j, 2] < bbox[i, j, 0]:
																															UpperCAmelCase_			:      List[str]     						=  bbox[i, j, 2]
																															UpperCAmelCase_			:      Tuple     						=  bbox[i, j, 0]
																															UpperCAmelCase_			:      Union[str, Any]     						=  t

													UpperCAmelCase_			:      int     						=  None
													if self.use_input_mask:
																			UpperCAmelCase_			:      Optional[int]     						=  ids_tensor([self.batch_size, self.seq_length]       ,     vocab_size=2   )

													UpperCAmelCase_			:      Optional[int]     						=  None
													if self.use_token_type_ids:
																			UpperCAmelCase_			:      str     						=  ids_tensor([self.batch_size, self.seq_length]       ,     self.type_vocab_size   )

													UpperCAmelCase_			:      Dict     						=  None
													UpperCAmelCase_			:      Tuple     						=  None
													if self.use_labels:
																			UpperCAmelCase_			:      Any     						=  ids_tensor([self.batch_size]       ,     self.type_sequence_label_size   )
																			UpperCAmelCase_			:      Dict     						=  ids_tensor([self.batch_size, self.seq_length]       ,     self.num_labels   )

													UpperCAmelCase_			:      int     						=  self.get_config()

													return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels

							def     UpperCAmelCase__	(							self							:     Any   )					->       List[Any]:







													"""simple docstring"""
													return LiltConfig(
													    vocab_size=self.vocab_size       ,     hidden_size=self.hidden_size       ,     num_hidden_layers=self.num_hidden_layers       ,     num_attention_heads=self.num_attention_heads       ,     intermediate_size=self.intermediate_size       ,     hidden_act=self.hidden_act       ,     hidden_dropout_prob=self.hidden_dropout_prob       ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,     max_position_embeddings=self.max_position_embeddings       ,     type_vocab_size=self.type_vocab_size       ,     initializer_range=self.initializer_range       ,     )

							def     UpperCAmelCase__	(							self							:     List[Any]       ,     __magic_name__							:     str       ,     __magic_name__							:     Optional[Any]       ,     __magic_name__							:     int       ,     __magic_name__							:     Optional[Any]       ,     __magic_name__							:     int       ,     __magic_name__							:     Optional[Any]       ,     __magic_name__							:     int       ,     )					->       Optional[int]:







													"""simple docstring"""
													UpperCAmelCase_			:      Any     						=  LiltModel(config=__magic_name__   )
													model.to(__magic_name__   )
													model.eval()
													UpperCAmelCase_			:      Optional[Any]     						=  model(__magic_name__       ,     bbox=__magic_name__       ,     attention_mask=__magic_name__       ,     token_type_ids=__magic_name__   )
													UpperCAmelCase_			:      List[Any]     						=  model(__magic_name__       ,     bbox=__magic_name__       ,     token_type_ids=__magic_name__   )
													UpperCAmelCase_			:      Optional[int]     						=  model(__magic_name__       ,     bbox=__magic_name__   )

													self.parent.assertEqual(result.last_hidden_state.shape       ,     (self.batch_size, self.seq_length, self.hidden_size)   )
													self.parent.assertEqual(result.pooler_output.shape       ,     (self.batch_size, self.hidden_size)   )

							def     UpperCAmelCase__	(							self							:     int       ,     __magic_name__							:     Optional[Any]       ,     __magic_name__							:     List[str]       ,     __magic_name__							:     Any       ,     __magic_name__							:     Optional[int]       ,     __magic_name__							:     str       ,     __magic_name__							:     Optional[int]       ,     __magic_name__							:     List[Any]       ,     )					->       Optional[Any]:







													"""simple docstring"""
													UpperCAmelCase_			:      Any     						=  self.num_labels
													UpperCAmelCase_			:      List[Any]     						=  LiltForTokenClassification(config=__magic_name__   )
													model.to(__magic_name__   )
													model.eval()
													UpperCAmelCase_			:      List[Any]     						=  model(
													    __magic_name__       ,     bbox=__magic_name__       ,     attention_mask=__magic_name__       ,     token_type_ids=__magic_name__       ,     labels=__magic_name__   )
													self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.seq_length, self.num_labels)   )

							def     UpperCAmelCase__	(							self							:     Any       ,     __magic_name__							:     Optional[Any]       ,     __magic_name__							:     Dict       ,     __magic_name__							:     Any       ,     __magic_name__							:     Optional[int]       ,     __magic_name__							:     int       ,     __magic_name__							:     Tuple       ,     __magic_name__							:     Any       ,     )					->       Optional[Any]:







													"""simple docstring"""
													UpperCAmelCase_			:      str     						=  LiltForQuestionAnswering(config=__magic_name__   )
													model.to(__magic_name__   )
													model.eval()
													UpperCAmelCase_			:      Optional[Any]     						=  model(
													    __magic_name__       ,     bbox=__magic_name__       ,     attention_mask=__magic_name__       ,     token_type_ids=__magic_name__       ,     start_positions=__magic_name__       ,     end_positions=__magic_name__       ,     )
													self.parent.assertEqual(result.start_logits.shape       ,     (self.batch_size, self.seq_length)   )
													self.parent.assertEqual(result.end_logits.shape       ,     (self.batch_size, self.seq_length)   )



							def     UpperCAmelCase__	(							self							:     List[Any]   )					->       List[Any]:







													"""simple docstring"""
													UpperCAmelCase_			:      Tuple     						=  self.prepare_config_and_inputs()
													(
													    (
													    UpperCAmelCase_
													)						,       (
													    UpperCAmelCase_
													)						,       (
													    UpperCAmelCase_
													)						,       (
													    UpperCAmelCase_
													)						,       (
													    UpperCAmelCase_
													)						,       (
													    UpperCAmelCase_
													)						,       (
													    UpperCAmelCase_
													)						,       
													)			:      Optional[int]     						=  config_and_inputs
													UpperCAmelCase_			:      Tuple     						=  {
													    '''input_ids''': input_ids,
													    '''bbox''': bbox,
													    '''token_type_ids''': token_type_ids,
													    '''attention_mask''': input_mask,
													}
													return config, inputs_dict





@require_torch
class 					__a      (lowerCamelCase       ,		lowerCamelCase       ,		lowerCamelCase       ,		unittest.TestCase     ):
							__a					:   Tuple									=       (
							    (
							        LiltModel,
							        LiltForSequenceClassification,
							        LiltForTokenClassification,
							        LiltForQuestionAnswering,
							    )
							    if is_torch_available()
							    else ()
							)
							__a					:   Any									=       (
							    {
							        "feature-extraction": LiltModel,
							        "question-answering": LiltForQuestionAnswering,
							        "text-classification": LiltForSequenceClassification,
							        "token-classification": LiltForTokenClassification,
							        "zero-shot": LiltForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							__a					:   Union[str, Any]									=       False
							__a					:   int									=       False

							def     UpperCAmelCase__	(							self							:     List[str]       ,     __magic_name__							:     Dict       ,     __magic_name__							:     List[Any]       ,     __magic_name__							:     Optional[int]       ,     __magic_name__							:     Optional[Any]       ,     __magic_name__							:     int   )					->       str:







													"""simple docstring"""
													return True

							def     UpperCAmelCase__	(							self							:     str   )					->       List[Any]:







													"""simple docstring"""
													UpperCAmelCase_			:      List[Any]     						=  LiltModelTester(self   )
													UpperCAmelCase_			:      List[Any]     						=  ConfigTester(self       ,     config_class=__magic_name__       ,     hidden_size=37   )

							def     UpperCAmelCase__	(							self							:     Union[str, Any]   )					->       str:







													"""simple docstring"""
													self.config_tester.run_common_tests()

							def     UpperCAmelCase__	(							self							:     str   )					->       Union[str, Any]:







													"""simple docstring"""
													UpperCAmelCase_			:      int     						=  self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*__magic_name__   )

							def     UpperCAmelCase__	(							self							:     str   )					->       str:







													"""simple docstring"""
													UpperCAmelCase_			:      Any     						=  self.model_tester.prepare_config_and_inputs()
													for type in ["absolute", "relative_key", "relative_key_query"]:
																			UpperCAmelCase_			:      Tuple     						=  type
																			self.model_tester.create_and_check_model(*__magic_name__   )

							def     UpperCAmelCase__	(							self							:     Union[str, Any]   )					->       int:







													"""simple docstring"""
													UpperCAmelCase_			:      Any     						=  self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_token_classification(*__magic_name__   )

							def     UpperCAmelCase__	(							self							:     str   )					->       List[Any]:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[Any]     						=  self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_question_answering(*__magic_name__   )



							@slow
							def     UpperCAmelCase__	(							self							:     int   )					->       Union[str, Any]:







													"""simple docstring"""
													for model_name in LILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																			UpperCAmelCase_			:      Optional[int]     						=  LiltModel.from_pretrained(__magic_name__   )
																			self.assertIsNotNone(__magic_name__   )





@require_torch
@slow
class 					__a      (unittest.TestCase     ):

							def     UpperCAmelCase__	(							self							:     Tuple   )					->       Tuple:







													"""simple docstring"""
													UpperCAmelCase_			:      str     						=  LiltModel.from_pretrained('''SCUT-DLVCLab/lilt-roberta-en-base'''   ).to(__magic_name__   )

													UpperCAmelCase_			:      Any     						=  torch.tensor([[1, 2]]       ,     device=__magic_name__   )
													UpperCAmelCase_			:      int     						=  torch.tensor([[[1, 2, 3, 4], [5, 6, 7, 8]]]       ,     device=__magic_name__   )

													# forward pass
													with torch.no_grad():
																			UpperCAmelCase_			:      Optional[int]     						=  model(input_ids=__magic_name__       ,     bbox=__magic_name__   )

													UpperCAmelCase_			:      int     						=  torch.Size([1, 2, 7_68]   )
													UpperCAmelCase_			:      List[str]     						=  torch.tensor(
													    [[-0.0_6_5_3, 0.0_9_5_0, -0.0_0_6_1], [-0.0_5_4_5, 0.0_9_2_6, -0.0_3_2_4]]       ,     device=__magic_name__       ,     )

													self.assertTrue(outputs.last_hidden_state.shape       ,     __magic_name__   )
													self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :, :3]       ,     __magic_name__       ,     atol=1E-3   )   )





