


"""simple docstring"""





import datasets

from .evaluate import evaluate


snake_case__		:     List[Any]    				=				'''\
@inproceedings{Rajpurkar2016SQuAD10,
  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},
  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},
  booktitle={EMNLP},
  year={2016}
}
'''

snake_case__		:     Tuple    				=				'''
This metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD).

Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by
crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,
from the corresponding reading passage, or the question might be unanswerable.
'''

snake_case__		:     List[Any]    				=				'''
Computes SQuAD scores (F1 and EM).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - \'id\': id of the question-answer pair as given in the references (see below)
        - \'prediction_text\': the text of the answer
    references: List of question-answers dictionaries with the following key-values:
        - \'id\': id of the question-answer pair (see above),
        - \'answers\': a Dict in the SQuAD dataset format
            {
                \'text\': list of possible texts for the answer, as a list of strings
                \'answer_start\': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    \'exact_match\': Exact match (the normalized answer exactly match the gold answer)
    \'f1\': The F-score of predicted tokens versus the gold answer
Examples:

    >>> predictions = [{\'prediction_text\': \'1976\', \'id\': \'56e10a3be3433e1400422b22\'}]
    >>> references = [{\'answers\': {\'answer_start\': [97], \'text\': [\'1976\']}, \'id\': \'56e10a3be3433e1400422b22\'}]
    >>> squad_metric = datasets.load_metric("squad")
    >>> results = squad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 100.0, \'f1\': 100.0}
'''



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,					_KWARGS_DESCRIPTION			)
class 		snake_case_(       datasets.Metric			):






     def      lowerCamelCase__     (	self  :					Tuple				):
           return datasets.MetricInfo(
               description=_DESCRIPTION					,				citation=_CITATION					,				inputs_description=_KWARGS_DESCRIPTION					,				features=datasets.Features(
                   {
                       '''predictions''': {'''id''': datasets.Value('''string'''				), '''prediction_text''': datasets.Value('''string'''				)},
                       '''references''': {
                           '''id''': datasets.Value('''string'''				),
                           '''answers''': datasets.features.Sequence(
                               {
                                   '''text''': datasets.Value('''string'''				),
                                   '''answer_start''': datasets.Value('''int32'''				),
                               }				),
                       },
                   }				)					,				codebase_urls=['''https://rajpurkar.github.io/SQuAD-explorer/''']					,				reference_urls=['''https://rajpurkar.github.io/SQuAD-explorer/''']					,				)






     def      lowerCamelCase__     (	self  :					Any					,				UpperCamelCase_  :					Any					,				UpperCamelCase_  :					Dict				):
           lowerCAmelCase					:	Optional[int]       =   {prediction['''id''']: prediction['''prediction_text'''] for prediction in predictions}
           lowerCAmelCase					:	List[Any]       =   [
               {
                   '''paragraphs''': [
                       {
                           '''qas''': [
                               {
                                   '''answers''': [{'''text''': answer_text} for answer_text in ref['''answers''']['''text''']],
                                   '''id''': ref['''id'''],
                               }
                               for ref in references
                           ]
                       }
                   ]
               }
           ]
           lowerCAmelCase					:	Union[str, Any]       =   evaluate(dataset=UpperCamelCase_					,				predictions=UpperCamelCase_				)
           return score






"""simple docstring"""





import time
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, torch_device

from ..test_modeling_common import ids_tensor


if is_torch_available():
 import torch

 from transformers.generation import (
     MaxLengthCriteria,
     MaxNewTokensCriteria,
     MaxTimeCriteria,
     StoppingCriteriaList,
     validate_stopping_criteria,
 )



@require_torch
class 		snake_case_(       unittest.TestCase			):






     def      lowerCamelCase__     (	self  :					int					,				UpperCamelCase_  :					int				):
           lowerCAmelCase					:	str       =   3
           lowerCAmelCase					:	Tuple       =   2_5_0

           lowerCAmelCase					:	Optional[Any]       =   ids_tensor((batch_size, length)					,				UpperCamelCase_				)
           lowerCAmelCase					:	Optional[Any]       =   torch.ones((batch_size, length)					,				device=UpperCamelCase_					,				dtype=torch.float				) / length
           return input_ids, scores






     def      lowerCamelCase__     (	self  :					Optional[Any]				):
           lowerCAmelCase,      lowerCAmelCase					:	Optional[int]       =   self._get_tensors(5				)

           lowerCAmelCase					:	Union[str, Any]       =   StoppingCriteriaList(
               [
                   MaxLengthCriteria(max_length=1_0				),
                   MaxTimeCriteria(max_time=0.1				),
               ]				)

           self.assertFalse(criteria(UpperCamelCase_					,				UpperCamelCase_				)				)

           lowerCAmelCase,      lowerCAmelCase					:	List[str]       =   self._get_tensors(9				)
           self.assertFalse(criteria(UpperCamelCase_					,				UpperCamelCase_				)				)

           lowerCAmelCase,      lowerCAmelCase					:	Any       =   self._get_tensors(1_0				)
           self.assertTrue(criteria(UpperCamelCase_					,				UpperCamelCase_				)				)






     def      lowerCamelCase__     (	self  :					Optional[Any]				):
           lowerCAmelCase					:	Optional[Any]       =   MaxLengthCriteria(max_length=1_0				)

           lowerCAmelCase,      lowerCAmelCase					:	Optional[Any]       =   self._get_tensors(5				)
           self.assertFalse(criteria(UpperCamelCase_					,				UpperCamelCase_				)				)

           lowerCAmelCase,      lowerCAmelCase					:	List[str]       =   self._get_tensors(9				)
           self.assertFalse(criteria(UpperCamelCase_					,				UpperCamelCase_				)				)

           lowerCAmelCase,      lowerCAmelCase					:	str       =   self._get_tensors(1_0				)
           self.assertTrue(criteria(UpperCamelCase_					,				UpperCamelCase_				)				)






     def      lowerCamelCase__     (	self  :					List[Any]				):
           lowerCAmelCase					:	Optional[Any]       =   MaxNewTokensCriteria(start_length=5					,				max_new_tokens=5				)

           lowerCAmelCase,      lowerCAmelCase					:	Optional[int]       =   self._get_tensors(5				)
           self.assertFalse(criteria(UpperCamelCase_					,				UpperCamelCase_				)				)

           lowerCAmelCase,      lowerCAmelCase					:	Union[str, Any]       =   self._get_tensors(9				)
           self.assertFalse(criteria(UpperCamelCase_					,				UpperCamelCase_				)				)

           lowerCAmelCase,      lowerCAmelCase					:	str       =   self._get_tensors(1_0				)
           self.assertTrue(criteria(UpperCamelCase_					,				UpperCamelCase_				)				)

           lowerCAmelCase					:	Dict       =   StoppingCriteriaList([criteria]				)
           self.assertEqual(criteria_list.max_length					,				1_0				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           lowerCAmelCase,      lowerCAmelCase					:	Tuple       =   self._get_tensors(5				)

           lowerCAmelCase					:	List[str]       =   MaxTimeCriteria(max_time=0.1				)
           self.assertFalse(criteria(UpperCamelCase_					,				UpperCamelCase_				)				)

           lowerCAmelCase					:	List[str]       =   MaxTimeCriteria(max_time=0.1					,				initial_timestamp=time.time() - 0.2				)
           self.assertTrue(criteria(UpperCamelCase_					,				UpperCamelCase_				)				)






     def      lowerCamelCase__     (	self  :					str				):
           validate_stopping_criteria(StoppingCriteriaList([MaxLengthCriteria(1_0				)]				)					,				1_0				)

           with self.assertWarns(UpperCamelCase_				):
                 validate_stopping_criteria(StoppingCriteriaList([MaxLengthCriteria(1_0				)]				)					,				1_1				)

           lowerCAmelCase					:	str       =   validate_stopping_criteria(StoppingCriteriaList()					,				1_1				)

           self.assertEqual(len(UpperCamelCase_				)					,				1				)


