





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_lowerCamelCase			       =      re.compile(r'\b(a|an|the)\b', re.UNICODE)

_lowerCamelCase			       =      None




def    __UpperCAmelCase(      ):
       _lowerCamelCase							:     Union[str, Any]	=		argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''      )
       parser.add_argument('''data_file''' ,    metavar='''data.json''' ,    help='''Input data JSON file.'''      )
       parser.add_argument('''pred_file''' ,    metavar='''pred.json''' ,    help='''Model predictions.'''      )
       parser.add_argument(
           '''--out-file''' ,    '''-o''' ,    metavar='''eval.json''' ,    help='''Write accuracy metrics to file (default is stdout).'''      )
       parser.add_argument(
           '''--na-prob-file''' ,    '''-n''' ,    metavar='''na_prob.json''' ,    help='''Model estimates of probability of no answer.'''      )
       parser.add_argument(
           '''--na-prob-thresh''' ,    '''-t''' ,    type=lowercase_ ,    default=1.0 ,    help='''Predict "" if no-answer probability exceeds this (default = 1.0).''' ,    )
       parser.add_argument(
           '''--out-image-dir''' ,    '''-p''' ,    metavar='''out_images''' ,    default=lowercase_ ,    help='''Save precision-recall curves to directory.'''      )
       parser.add_argument('''--verbose''' ,    '''-v''' ,    action='''store_true'''      )
       if len(sys.argv      ) == 1:
              parser.print_help()
              sys.exit(1      )
       return parser.parse_args()




def    __UpperCAmelCase(      lowercase_      ):
       _lowerCamelCase							:     Union[str, Any]	=		{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            _lowerCamelCase							:     Union[str, Any]	=		bool(qa['''answers''']['''text''']      )
       return qid_to_has_ans




def    __UpperCAmelCase(      lowercase_      ):

       def remove_articles(lowercase_      ):
              return ARTICLES_REGEX.sub(''' ''' ,    lowercase_      )

       def white_space_fix(lowercase_      ):
              return " ".join(text.split()      )

       def remove_punc(lowercase_      ):
              _lowerCamelCase							:     List[str]	=		set(string.punctuation      )
              return "".join(ch for ch in text if ch not in exclude      )

       def lower(lowercase_      ):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(lowercase_      )      )      )      )




def    __UpperCAmelCase(      lowercase_      ):
       if not s:
              return []
       return normalize_answer(lowercase_      ).split()




def    __UpperCAmelCase(      lowercase_ ,    lowercase_      ):
       return int(normalize_answer(lowercase_      ) == normalize_answer(lowercase_      )      )




def    __UpperCAmelCase(      lowercase_ ,    lowercase_      ):
       _lowerCamelCase							:     int	=		get_tokens(lowercase_      )
       _lowerCamelCase							:     List[Any]	=		get_tokens(lowercase_      )
       _lowerCamelCase							:     Optional[int]	=		collections.Counter(lowercase_      ) & collections.Counter(lowercase_      )
       _lowerCamelCase							:     str	=		sum(common.values()      )
       if len(lowercase_      ) == 0 or len(lowercase_      ) == 0:
              # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
              return int(gold_toks == pred_toks      )
       if num_same == 0:
              return 0
       _lowerCamelCase							:     int	=		1.0 * num_same / len(lowercase_      )
       _lowerCamelCase							:     Optional[Any]	=		1.0 * num_same / len(lowercase_      )
       _lowerCamelCase							:     Tuple	=		(2 * precision * recall) / (precision + recall)
       return fa




def    __UpperCAmelCase(      lowercase_ ,    lowercase_      ):
       _lowerCamelCase							:     int	=		{}
       _lowerCamelCase							:     str	=		{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            _lowerCamelCase							:     Dict	=		qa['''id''']
                            _lowerCamelCase							:     Tuple	=		[t for t in qa['''answers''']['''text'''] if normalize_answer(lowercase_      )]
                            if not gold_answers:
                                   # For unanswerable questions, only correct answer is empty string
                                   _lowerCamelCase							:     Tuple	=		['''''']
                            if qid not in preds:
                                   print(F"""Missing prediction for {qid}"""      )
                                   continue
                            _lowerCamelCase							:     List[str]	=		preds[qid]
                            # Take max over all gold answers
                            _lowerCamelCase							:     Any	=		max(compute_exact(lowercase_ ,    lowercase_      ) for a in gold_answers      )
                            _lowerCamelCase							:     Optional[int]	=		max(compute_fa(lowercase_ ,    lowercase_      ) for a in gold_answers      )
       return exact_scores, fa_scores




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      ):
       _lowerCamelCase							:     List[str]	=		{}
       for qid, s in scores.items():
              _lowerCamelCase							:     int	=		na_probs[qid] > na_prob_thresh
              if pred_na:
                     _lowerCamelCase							:     Optional[int]	=		float(not qid_to_has_ans[qid]      )
              else:
                     _lowerCamelCase							:     List[str]	=		s
       return new_scores




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_=None      ):
       if not qid_list:
              _lowerCamelCase							:     Optional[int]	=		len(lowercase_      )
              return collections.OrderedDict(
                  [
                      ('''exact''', 1_0_0.0 * sum(exact_scores.values()      ) / total),
                      ('''f1''', 1_0_0.0 * sum(fa_scores.values()      ) / total),
                      ('''total''', total),
                  ]      )
       else:
              _lowerCamelCase							:     int	=		len(lowercase_      )
              return collections.OrderedDict(
                  [
                      ('''exact''', 1_0_0.0 * sum(exact_scores[k] for k in qid_list      ) / total),
                      ('''f1''', 1_0_0.0 * sum(fa_scores[k] for k in qid_list      ) / total),
                      ('''total''', total),
                  ]      )




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_      ):
       for k in new_eval:
              _lowerCamelCase							:     Union[str, Any]	=		new_eval[k]




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      ):
       plt.step(lowercase_ ,    lowercase_ ,    color='''b''' ,    alpha=0.2 ,    where='''post'''      )
       plt.fill_between(lowercase_ ,    lowercase_ ,    step='''post''' ,    alpha=0.2 ,    color='''b'''      )
       plt.xlabel('''Recall'''      )
       plt.ylabel('''Precision'''      )
       plt.xlim([0.0, 1.0_5]      )
       plt.ylim([0.0, 1.0_5]      )
       plt.title(lowercase_      )
       plt.savefig(lowercase_      )
       plt.clf()




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_=None ,    lowercase_=None      ):
       _lowerCamelCase							:     Optional[int]	=		sorted(lowercase_ ,    key=lambda lowercase_      : na_probs[k]      )
       _lowerCamelCase							:     Any	=		0.0
       _lowerCamelCase							:     Dict	=		1.0
       _lowerCamelCase							:     str	=		0.0
       _lowerCamelCase							:     Tuple	=		[1.0]
       _lowerCamelCase							:     Optional[Any]	=		[0.0]
       _lowerCamelCase							:     str	=		0.0
       for i, qid in enumerate(lowercase_      ):
              if qid_to_has_ans[qid]:
                     true_pos += scores[qid]
              _lowerCamelCase							:     Dict	=		true_pos / float(i + 1      )
              _lowerCamelCase							:     List[Any]	=		true_pos / float(lowercase_      )
              if i == len(lowercase_      ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                     # i.e., if we can put a threshold after this point
                     avg_prec += cur_p * (cur_r - recalls[-1])
                     precisions.append(lowercase_      )
                     recalls.append(lowercase_      )
       if out_image:
              plot_pr_curve(lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      )
       return {"ap": 1_0_0.0 * avg_prec}




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      ):
       if out_image_dir and not os.path.exists(lowercase_      ):
              os.makedirs(lowercase_      )
       _lowerCamelCase							:     List[Any]	=		sum(1 for v in qid_to_has_ans.values() if v      )
       if num_true_pos == 0:
              return
       _lowerCamelCase							:     List[str]	=		make_precision_recall_eval(
           lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    out_image=os.path.join(lowercase_ ,    '''pr_exact.png'''      ) ,    title='''Precision-Recall curve for Exact Match score''' ,    )
       _lowerCamelCase							:     Optional[int]	=		make_precision_recall_eval(
           lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    out_image=os.path.join(lowercase_ ,    '''pr_f1.png'''      ) ,    title='''Precision-Recall curve for F1 score''' ,    )
       _lowerCamelCase							:     List[str]	=		{k: float(lowercase_      ) for k, v in qid_to_has_ans.items()}
       _lowerCamelCase							:     List[str]	=		make_precision_recall_eval(
           lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    out_image=os.path.join(lowercase_ ,    '''pr_oracle.png'''      ) ,    title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)''' ,    )
       merge_eval(lowercase_ ,    lowercase_ ,    '''pr_exact'''      )
       merge_eval(lowercase_ ,    lowercase_ ,    '''pr_f1'''      )
       merge_eval(lowercase_ ,    lowercase_ ,    '''pr_oracle'''      )




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      ):
       if not qid_list:
              return
       _lowerCamelCase							:     int	=		[na_probs[k] for k in qid_list]
       _lowerCamelCase							:     Optional[int]	=		np.ones_like(lowercase_      ) / float(len(lowercase_      )      )
       plt.hist(lowercase_ ,    weights=lowercase_ ,    bins=20 ,    range=(0.0, 1.0)      )
       plt.xlabel('''Model probability of no-answer'''      )
       plt.ylabel('''Proportion of dataset'''      )
       plt.title(F"""Histogram of no-answer probability: {name}"""      )
       plt.savefig(os.path.join(lowercase_ ,    F"""na_prob_hist_{name}.png"""      )      )
       plt.clf()




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      ):
       _lowerCamelCase							:     Optional[int]	=		sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]      )
       _lowerCamelCase							:     str	=		num_no_ans
       _lowerCamelCase							:     Tuple	=		cur_score
       _lowerCamelCase							:     Union[str, Any]	=		0.0
       _lowerCamelCase							:     Tuple	=		sorted(lowercase_ ,    key=lambda lowercase_      : na_probs[k]      )
       for i, qid in enumerate(lowercase_      ):
              if qid not in scores:
                     continue
              if qid_to_has_ans[qid]:
                     _lowerCamelCase							:     Any	=		scores[qid]
              else:
                     if preds[qid]:
                            _lowerCamelCase							:     Any	=		-1
                     else:
                            _lowerCamelCase							:     Tuple	=		0
              cur_score += diff
              if cur_score > best_score:
                     _lowerCamelCase							:     Union[str, Any]	=		cur_score
                     _lowerCamelCase							:     Tuple	=		na_probs[qid]
       return 1_0_0.0 * best_score / len(lowercase_      ), best_thresh




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      ):
       _lowerCamelCase,  _lowerCamelCase							:     Optional[int]	=		find_best_thresh(lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      )
       _lowerCamelCase,  _lowerCamelCase							:     List[Any]	=		find_best_thresh(lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      )
       _lowerCamelCase							:     List[Any]	=		best_exact
       _lowerCamelCase							:     int	=		exact_thresh
       _lowerCamelCase							:     Optional[Any]	=		best_fa
       _lowerCamelCase							:     List[str]	=		fa_thresh




def    __UpperCAmelCase(      ):
       with open(OPTS.data_file      ) as f:
              _lowerCamelCase							:     Any	=		json.load(lowercase_      )
              _lowerCamelCase							:     Dict	=		dataset_json['''data''']
       with open(OPTS.pred_file      ) as f:
              _lowerCamelCase							:     Tuple	=		json.load(lowercase_      )
       if OPTS.na_prob_file:
              with open(OPTS.na_prob_file      ) as f:
                     _lowerCamelCase							:     Tuple	=		json.load(lowercase_      )
       else:
              _lowerCamelCase							:     str	=		{k: 0.0 for k in preds}
       _lowerCamelCase							:     List[Any]	=		make_qid_to_has_ans(lowercase_      )  # maps qid to True/False
       _lowerCamelCase							:     Optional[int]	=		[k for k, v in qid_to_has_ans.items() if v]
       _lowerCamelCase							:     Union[str, Any]	=		[k for k, v in qid_to_has_ans.items() if not v]
       _lowerCamelCase,  _lowerCamelCase							:     str	=		get_raw_scores(lowercase_ ,    lowercase_      )
       _lowerCamelCase							:     Optional[int]	=		apply_no_ans_threshold(lowercase_ ,    lowercase_ ,    lowercase_ ,    OPTS.na_prob_thresh      )
       _lowerCamelCase							:     Tuple	=		apply_no_ans_threshold(lowercase_ ,    lowercase_ ,    lowercase_ ,    OPTS.na_prob_thresh      )
       _lowerCamelCase							:     Dict	=		make_eval_dict(lowercase_ ,    lowercase_      )
       if has_ans_qids:
              _lowerCamelCase							:     Dict	=		make_eval_dict(lowercase_ ,    lowercase_ ,    qid_list=lowercase_      )
              merge_eval(lowercase_ ,    lowercase_ ,    '''HasAns'''      )
       if no_ans_qids:
              _lowerCamelCase							:     Tuple	=		make_eval_dict(lowercase_ ,    lowercase_ ,    qid_list=lowercase_      )
              merge_eval(lowercase_ ,    lowercase_ ,    '''NoAns'''      )
       if OPTS.na_prob_file:
              find_all_best_thresh(lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      )
       if OPTS.na_prob_file and OPTS.out_image_dir:
              run_precision_recall_analysis(lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    OPTS.out_image_dir      )
              histogram_na_prob(lowercase_ ,    lowercase_ ,    OPTS.out_image_dir ,    '''hasAns'''      )
              histogram_na_prob(lowercase_ ,    lowercase_ ,    OPTS.out_image_dir ,    '''noAns'''      )
       if OPTS.out_file:
              with open(OPTS.out_file ,    '''w'''      ) as f:
                     json.dump(lowercase_ ,    lowercase_      )
       else:
              print(json.dumps(lowercase_ ,    indent=2      )      )


if __name__ == "__main__":
       _lowerCamelCase			       =      parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use('Agg')
              import matplotlib.pyplot as plt
       main()








_lowerCamelCase			       =      [4, 1, 7, 4, 2, 6, 4, 1, 5, 3, 7, 5]
_lowerCamelCase			       =      [3, 7, 7, 4, 2, 6, 4, 1, 5, 3, 7, 5]
_lowerCamelCase			       =      {
    0: 'Sunday',
    1: 'Monday',
    2: 'Tuesday',
    3: 'Wednesday',
    4: 'Thursday',
    5: 'Friday',
    6: 'Saturday',
}




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_      ):
       assert len(str(lowercase_      )      ) > 2, "year should be in YYYY format"
       assert 1 <= month <= 12, "month should be between 1 to 12"
       assert 1 <= day <= 31, "day should be between 1 to 31"

       # Doomsday algorithm:
       _lowerCamelCase							:     List[Any]	=		year // 1_00
       _lowerCamelCase							:     Dict	=		(5 * (century % 4) + 2) % 7
       _lowerCamelCase							:     Tuple	=		year % 1_00
       _lowerCamelCase							:     int	=		centurian % 12
       _lowerCamelCase							:     List[Any]	=		(
           (centurian // 12) + centurian_m + (centurian_m // 4) + century_anchor
       ) % 7
       _lowerCamelCase							:     List[Any]	=		(
           DOOMSDAY_NOT_LEAP[month - 1]
           if (year % 4 != 0) or (centurian == 0 and (year % 4_00) == 0)
           else DOOMSDAY_LEAP[month - 1]
       )
       _lowerCamelCase							:     Any	=		(dooms_day + day - day_anchor) % 7
       return WEEK_DAY_NAMES[week_day]


if __name__ == "__main__":
       import doctest

       doctest.testmod()

