

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCamelCase__  :      Dict			=    re.compile(r"\b(a|an|the)\b", re.UNICODE)

UpperCamelCase__  :      Optional[int]			=    None


def 					_UpperCAmelCase       (  ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'       )
    parser.add_argument('data_file' ,   metavar='data.json' ,   help='Input data JSON file.'       )
    parser.add_argument('pred_file' ,   metavar='pred.json' ,   help='Model predictions.'       )
    parser.add_argument(
        '--out-file' ,   '-o' ,   metavar='eval.json' ,   help='Write accuracy metrics to file (default is stdout).'       )
    parser.add_argument(
        '--na-prob-file' ,   '-n' ,   metavar='na_prob.json' ,   help='Model estimates of probability of no answer.'       )
    parser.add_argument(
        '--na-prob-thresh' ,   '-t' ,   type=_SCREAMING_SNAKE_CASE ,   default=1.0 ,   help='Predict "" if no-answer probability exceeds this (default = 1.0).' ,   )
    parser.add_argument(
        '--out-image-dir' ,   '-p' ,   metavar='out_images' ,   default=_SCREAMING_SNAKE_CASE ,   help='Save precision-recall curves to directory.'       )
    parser.add_argument('--verbose' ,   '-v' ,   action='store_true'       )
    if len(sys.argv       ) == 1:
        parser.print_help()
        sys.exit(1       )
    return parser.parse_args()


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Union[str, Any]       ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						{}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                SCREAMING_SNAKE_CASE_									=						bool(qa['answers']['text']       )
    return qid_to_has_ans


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Union[str, Any]       ):





    """simple docstring"""


    def remove_articles(_SCREAMING_SNAKE_CASE					:		Union[str, Any]       ):
        return ARTICLES_REGEX.sub(' ' ,   _SCREAMING_SNAKE_CASE       )

    def white_space_fix(_SCREAMING_SNAKE_CASE					:		str       ):
        return " ".join(text.split()       )

    def remove_punc(_SCREAMING_SNAKE_CASE					:		List[str]       ):
        SCREAMING_SNAKE_CASE_									=						set(string.punctuation       )
        return "".join(ch for ch in text if ch not in exclude       )

    def lower(_SCREAMING_SNAKE_CASE					:		str       ):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(_SCREAMING_SNAKE_CASE       )       )       )       )


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		List[Any]       ):





    """simple docstring"""


    if not s:
        return []
    return normalize_answer(_SCREAMING_SNAKE_CASE       ).split()


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		str ,   _SCREAMING_SNAKE_CASE					:		Any       ):





    """simple docstring"""


    return int(normalize_answer(_SCREAMING_SNAKE_CASE       ) == normalize_answer(_SCREAMING_SNAKE_CASE       )       )


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Optional[Any] ,   _SCREAMING_SNAKE_CASE					:		Union[str, Any]       ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						get_tokens(_SCREAMING_SNAKE_CASE       )
    SCREAMING_SNAKE_CASE_									=						get_tokens(_SCREAMING_SNAKE_CASE       )
    SCREAMING_SNAKE_CASE_									=						collections.Counter(_SCREAMING_SNAKE_CASE       ) & collections.Counter(_SCREAMING_SNAKE_CASE       )
    SCREAMING_SNAKE_CASE_									=						sum(common.values()       )
    if len(_SCREAMING_SNAKE_CASE       ) == 0 or len(_SCREAMING_SNAKE_CASE       ) == 0:
        # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
        return int(gold_toks == pred_toks       )
    if num_same == 0:
        return 0
    SCREAMING_SNAKE_CASE_									=						1.0 * num_same / len(_SCREAMING_SNAKE_CASE       )
    SCREAMING_SNAKE_CASE_									=						1.0 * num_same / len(_SCREAMING_SNAKE_CASE       )
    SCREAMING_SNAKE_CASE_									=						(2 * precision * recall) / (precision + recall)
    return fa


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Optional[int] ,   _SCREAMING_SNAKE_CASE					:		List[Any]       ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						{}
    SCREAMING_SNAKE_CASE_									=						{}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                SCREAMING_SNAKE_CASE_									=						qa['id']
                SCREAMING_SNAKE_CASE_									=						[t for t in qa['answers']['text'] if normalize_answer(_SCREAMING_SNAKE_CASE       )]
                if not gold_answers:
                    # For unanswerable questions, only correct answer is empty string
                    SCREAMING_SNAKE_CASE_									=						['']
                if qid not in preds:
                    print(f"""Missing prediction for {qid}"""       )
                    continue
                SCREAMING_SNAKE_CASE_									=						preds[qid]
                # Take max over all gold answers
                SCREAMING_SNAKE_CASE_									=						max(compute_exact(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       ) for a in gold_answers       )
                SCREAMING_SNAKE_CASE_									=						max(compute_fa(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       ) for a in gold_answers       )
    return exact_scores, fa_scores


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Optional[Any] ,   _SCREAMING_SNAKE_CASE					:		Optional[int] ,   _SCREAMING_SNAKE_CASE					:		List[Any] ,   _SCREAMING_SNAKE_CASE					:		Optional[int]       ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						{}
    for qid, s in scores.items():
        SCREAMING_SNAKE_CASE_									=						na_probs[qid] > na_prob_thresh
        if pred_na:
            SCREAMING_SNAKE_CASE_									=						float(not qid_to_has_ans[qid]       )
        else:
            SCREAMING_SNAKE_CASE_									=						s
    return new_scores


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		int ,   _SCREAMING_SNAKE_CASE					:		Optional[Any] ,   _SCREAMING_SNAKE_CASE					:		List[Any]=None       ):





    """simple docstring"""


    if not qid_list:
        SCREAMING_SNAKE_CASE_									=						len(_SCREAMING_SNAKE_CASE       )
        return collections.OrderedDict(
            [
                ('exact', 100.0 * sum(exact_scores.values()       ) / total),
                ('f1', 100.0 * sum(fa_scores.values()       ) / total),
                ('total', total),
            ]       )
    else:
        SCREAMING_SNAKE_CASE_									=						len(_SCREAMING_SNAKE_CASE       )
        return collections.OrderedDict(
            [
                ('exact', 100.0 * sum(exact_scores[k] for k in qid_list       ) / total),
                ('f1', 100.0 * sum(fa_scores[k] for k in qid_list       ) / total),
                ('total', total),
            ]       )


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Optional[int] ,   _SCREAMING_SNAKE_CASE					:		Optional[int] ,   _SCREAMING_SNAKE_CASE					:		Any       ):





    """simple docstring"""


    for k in new_eval:
        SCREAMING_SNAKE_CASE_									=						new_eval[k]


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		List[Any] ,   _SCREAMING_SNAKE_CASE					:		Dict ,   _SCREAMING_SNAKE_CASE					:		str ,   _SCREAMING_SNAKE_CASE					:		Optional[int]       ):





    """simple docstring"""


    plt.step(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   color='b' ,   alpha=0.2 ,   where='post'       )
    plt.fill_between(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   step='post' ,   alpha=0.2 ,   color='b'       )
    plt.xlabel('Recall'       )
    plt.ylabel('Precision'       )
    plt.xlim([0.0, 1.05]       )
    plt.ylim([0.0, 1.05]       )
    plt.title(_SCREAMING_SNAKE_CASE       )
    plt.savefig(_SCREAMING_SNAKE_CASE       )
    plt.clf()


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Tuple ,   _SCREAMING_SNAKE_CASE					:		Any ,   _SCREAMING_SNAKE_CASE					:		Any ,   _SCREAMING_SNAKE_CASE					:		int ,   _SCREAMING_SNAKE_CASE					:		Any=None ,   _SCREAMING_SNAKE_CASE					:		List[Any]=None       ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						sorted(_SCREAMING_SNAKE_CASE ,   key=lambda _SCREAMING_SNAKE_CASE       : na_probs[k]       )
    SCREAMING_SNAKE_CASE_									=						0.0
    SCREAMING_SNAKE_CASE_									=						1.0
    SCREAMING_SNAKE_CASE_									=						0.0
    SCREAMING_SNAKE_CASE_									=						[1.0]
    SCREAMING_SNAKE_CASE_									=						[0.0]
    SCREAMING_SNAKE_CASE_									=						0.0
    for i, qid in enumerate(_SCREAMING_SNAKE_CASE       ):
        if qid_to_has_ans[qid]:
            true_pos += scores[qid]
        SCREAMING_SNAKE_CASE_									=						true_pos / float(i + 1       )
        SCREAMING_SNAKE_CASE_									=						true_pos / float(_SCREAMING_SNAKE_CASE       )
        if i == len(_SCREAMING_SNAKE_CASE       ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
            # i.e., if we can put a threshold after this point
            avg_prec += cur_p * (cur_r - recalls[-1])
            precisions.append(_SCREAMING_SNAKE_CASE       )
            recalls.append(_SCREAMING_SNAKE_CASE       )
    if out_image:
        plot_pr_curve(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )
    return {"ap": 100.0 * avg_prec}


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		List[Any] ,   _SCREAMING_SNAKE_CASE					:		Dict ,   _SCREAMING_SNAKE_CASE					:		Tuple ,   _SCREAMING_SNAKE_CASE					:		Optional[Any] ,   _SCREAMING_SNAKE_CASE					:		Optional[int] ,   _SCREAMING_SNAKE_CASE					:		Tuple       ):





    """simple docstring"""


    if out_image_dir and not os.path.exists(_SCREAMING_SNAKE_CASE       ):
        os.makedirs(_SCREAMING_SNAKE_CASE       )
    SCREAMING_SNAKE_CASE_									=						sum(1 for v in qid_to_has_ans.values() if v       )
    if num_true_pos == 0:
        return
    SCREAMING_SNAKE_CASE_									=						make_precision_recall_eval(
        _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   out_image=os.path.join(_SCREAMING_SNAKE_CASE ,   'pr_exact.png'       ) ,   title='Precision-Recall curve for Exact Match score' ,   )
    SCREAMING_SNAKE_CASE_									=						make_precision_recall_eval(
        _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   out_image=os.path.join(_SCREAMING_SNAKE_CASE ,   'pr_f1.png'       ) ,   title='Precision-Recall curve for F1 score' ,   )
    SCREAMING_SNAKE_CASE_									=						{k: float(_SCREAMING_SNAKE_CASE       ) for k, v in qid_to_has_ans.items()}
    SCREAMING_SNAKE_CASE_									=						make_precision_recall_eval(
        _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   out_image=os.path.join(_SCREAMING_SNAKE_CASE ,   'pr_oracle.png'       ) ,   title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)' ,   )
    merge_eval(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   'pr_exact'       )
    merge_eval(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   'pr_f1'       )
    merge_eval(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   'pr_oracle'       )


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Optional[int] ,   _SCREAMING_SNAKE_CASE					:		Any ,   _SCREAMING_SNAKE_CASE					:		Tuple ,   _SCREAMING_SNAKE_CASE					:		int       ):





    """simple docstring"""


    if not qid_list:
        return
    SCREAMING_SNAKE_CASE_									=						[na_probs[k] for k in qid_list]
    SCREAMING_SNAKE_CASE_									=						np.ones_like(_SCREAMING_SNAKE_CASE       ) / float(len(_SCREAMING_SNAKE_CASE       )       )
    plt.hist(_SCREAMING_SNAKE_CASE ,   weights=_SCREAMING_SNAKE_CASE ,   bins=20 ,   range=(0.0, 1.0)       )
    plt.xlabel('Model probability of no-answer'       )
    plt.ylabel('Proportion of dataset'       )
    plt.title(f"""Histogram of no-answer probability: {name}"""       )
    plt.savefig(os.path.join(_SCREAMING_SNAKE_CASE ,   f"""na_prob_hist_{name}.png"""       )       )
    plt.clf()


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Union[str, Any] ,   _SCREAMING_SNAKE_CASE					:		List[Any] ,   _SCREAMING_SNAKE_CASE					:		str ,   _SCREAMING_SNAKE_CASE					:		Optional[int]       ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]       )
    SCREAMING_SNAKE_CASE_									=						num_no_ans
    SCREAMING_SNAKE_CASE_									=						cur_score
    SCREAMING_SNAKE_CASE_									=						0.0
    SCREAMING_SNAKE_CASE_									=						sorted(_SCREAMING_SNAKE_CASE ,   key=lambda _SCREAMING_SNAKE_CASE       : na_probs[k]       )
    for i, qid in enumerate(_SCREAMING_SNAKE_CASE       ):
        if qid not in scores:
            continue
        if qid_to_has_ans[qid]:
            SCREAMING_SNAKE_CASE_									=						scores[qid]
        else:
            if preds[qid]:
                SCREAMING_SNAKE_CASE_									=						-1
            else:
                SCREAMING_SNAKE_CASE_									=						0
        cur_score += diff
        if cur_score > best_score:
            SCREAMING_SNAKE_CASE_									=						cur_score
            SCREAMING_SNAKE_CASE_									=						na_probs[qid]
    return 100.0 * best_score / len(_SCREAMING_SNAKE_CASE       ), best_thresh


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		List[Any] ,   _SCREAMING_SNAKE_CASE					:		int ,   _SCREAMING_SNAKE_CASE					:		Union[str, Any] ,   _SCREAMING_SNAKE_CASE					:		Optional[int] ,   _SCREAMING_SNAKE_CASE					:		int ,   _SCREAMING_SNAKE_CASE					:		str       ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_									=						find_best_thresh(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )
    SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_									=						find_best_thresh(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )
    SCREAMING_SNAKE_CASE_									=						best_exact
    SCREAMING_SNAKE_CASE_									=						exact_thresh
    SCREAMING_SNAKE_CASE_									=						best_fa
    SCREAMING_SNAKE_CASE_									=						fa_thresh


def 					_UpperCAmelCase       (  ):





    """simple docstring"""


    with open(OPTS.data_file       ) as f:
        SCREAMING_SNAKE_CASE_									=						json.load(_SCREAMING_SNAKE_CASE       )
        SCREAMING_SNAKE_CASE_									=						dataset_json['data']
    with open(OPTS.pred_file       ) as f:
        SCREAMING_SNAKE_CASE_									=						json.load(_SCREAMING_SNAKE_CASE       )
    if OPTS.na_prob_file:
        with open(OPTS.na_prob_file       ) as f:
            SCREAMING_SNAKE_CASE_									=						json.load(_SCREAMING_SNAKE_CASE       )
    else:
        SCREAMING_SNAKE_CASE_									=						{k: 0.0 for k in preds}
    SCREAMING_SNAKE_CASE_									=						make_qid_to_has_ans(_SCREAMING_SNAKE_CASE       )  # maps qid to True/False
    SCREAMING_SNAKE_CASE_									=						[k for k, v in qid_to_has_ans.items() if v]
    SCREAMING_SNAKE_CASE_									=						[k for k, v in qid_to_has_ans.items() if not v]
    SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_									=						get_raw_scores(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )
    SCREAMING_SNAKE_CASE_									=						apply_no_ans_threshold(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   OPTS.na_prob_thresh       )
    SCREAMING_SNAKE_CASE_									=						apply_no_ans_threshold(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   OPTS.na_prob_thresh       )
    SCREAMING_SNAKE_CASE_									=						make_eval_dict(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )
    if has_ans_qids:
        SCREAMING_SNAKE_CASE_									=						make_eval_dict(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   qid_list=_SCREAMING_SNAKE_CASE       )
        merge_eval(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   'HasAns'       )
    if no_ans_qids:
        SCREAMING_SNAKE_CASE_									=						make_eval_dict(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   qid_list=_SCREAMING_SNAKE_CASE       )
        merge_eval(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   'NoAns'       )
    if OPTS.na_prob_file:
        find_all_best_thresh(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )
    if OPTS.na_prob_file and OPTS.out_image_dir:
        run_precision_recall_analysis(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   OPTS.out_image_dir       )
        histogram_na_prob(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   OPTS.out_image_dir ,   'hasAns'       )
        histogram_na_prob(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   OPTS.out_image_dir ,   'noAns'       )
    if OPTS.out_file:
        with open(OPTS.out_file ,   'w'       ) as f:
            json.dump(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )
    else:
        print(json.dumps(_SCREAMING_SNAKE_CASE ,   indent=2       )       )


if __name__ == "__main__":
       UpperCamelCase__  :      int			=    parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use("Agg")
              import matplotlib.pyplot as plt
       main()





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


UpperCamelCase__  :      Any			=    {
    "configuration_mvp": ["MVP_PRETRAINED_CONFIG_ARCHIVE_MAP", "MvpConfig", "MvpOnnxConfig"],
    "tokenization_mvp": ["MvpTokenizer"],
}

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCamelCase__  :      Optional[int]			=    ["MvpTokenizerFast"]

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCamelCase__  :      str			=    [
           "MVP_PRETRAINED_MODEL_ARCHIVE_LIST",
           "MvpForCausalLM",
           "MvpForConditionalGeneration",
           "MvpForQuestionAnswering",
           "MvpForSequenceClassification",
           "MvpModel",
           "MvpPreTrainedModel",
       ]

if TYPE_CHECKING:
       from .configuration_mvp import MVP_PRETRAINED_CONFIG_ARCHIVE_MAP, MvpConfig, MvpOnnxConfig
       from .tokenization_mvp import MvpTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_mvp_fast import MvpTokenizerFast

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_mvp import (
                  MVP_PRETRAINED_MODEL_ARCHIVE_LIST,
                  MvpForCausalLM,
                  MvpForConditionalGeneration,
                  MvpForQuestionAnswering,
                  MvpForSequenceClassification,
                  MvpModel,
                  MvpPreTrainedModel,
              )

else:
       import sys

       UpperCamelCase__  :      Dict			=    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


