



import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCAmelCase__     = re.compile(r"\b(a|an|the)\b", re.UNICODE)

UpperCAmelCase__     = None





def 							A			(  )       ->				List[str]:







			'''simple docstring'''
			_UpperCAmelCase	 =     argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'					)
			parser.add_argument('data_file'							,     metavar='data.json'							,     help='Input data JSON file.'					)
			parser.add_argument('pred_file'							,     metavar='pred.json'							,     help='Model predictions.'					)
			parser.add_argument(
			    '--out-file'							,     '-o'							,     metavar='eval.json'							,     help='Write accuracy metrics to file (default is stdout).'					)
			parser.add_argument(
			    '--na-prob-file'							,     '-n'							,     metavar='na_prob.json'							,     help='Model estimates of probability of no answer.'					)
			parser.add_argument(
			    '--na-prob-thresh'							,     '-t'							,     type=_UpperCAmelCase							,     default=1.0							,     help='Predict "" if no-answer probability exceeds this (default = 1.0).'							,     )
			parser.add_argument(
			    '--out-image-dir'							,     '-p'							,     metavar='out_images'							,     default=_UpperCAmelCase							,     help='Save precision-recall curves to directory.'					)
			parser.add_argument('--verbose'							,     '-v'							,     action='store_true'					)
			if len(sys.argv					) == 1:
						parser.print_help()
						sys.exit(1					)
			return parser.parse_args()





def 							A			(  _UpperCAmelCase       :     List[str]					)       ->				Dict:







			'''simple docstring'''
			_UpperCAmelCase	 =     {}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												_UpperCAmelCase	 =     bool(qa['answers']['text']					)
			return qid_to_has_ans





def 							A			(  _UpperCAmelCase       :     Union[str, Any]					)       ->				str:







			'''simple docstring'''

			def remove_articles(_UpperCAmelCase       :     List[str]					):
						return ARTICLES_REGEX.sub(' '							,     _UpperCAmelCase					)

			def white_space_fix(_UpperCAmelCase       :     Optional[int]					):
						return " ".join(text.split()					)

			def remove_punc(_UpperCAmelCase       :     List[str]					):
						_UpperCAmelCase	 =     set(string.punctuation					)
						return "".join(ch for ch in text if ch not in exclude					)

			def lower(_UpperCAmelCase       :     Optional[int]					):
						return text.lower()

			return white_space_fix(remove_articles(remove_punc(lower(_UpperCAmelCase					)					)					)					)





def 							A			(  _UpperCAmelCase       :     Optional[Any]					)       ->				List[str]:







			'''simple docstring'''
			if not s:
						return []
			return normalize_answer(_UpperCAmelCase					).split()





def 							A			(  _UpperCAmelCase       :     Any							,     _UpperCAmelCase       :     Tuple					)       ->				Dict:







			'''simple docstring'''
			return int(normalize_answer(_UpperCAmelCase					) == normalize_answer(_UpperCAmelCase					)					)





def 							A			(  _UpperCAmelCase       :     List[Any]							,     _UpperCAmelCase       :     List[str]					)       ->				List[Any]:







			'''simple docstring'''
			_UpperCAmelCase	 =     get_tokens(_UpperCAmelCase					)
			_UpperCAmelCase	 =     get_tokens(_UpperCAmelCase					)
			_UpperCAmelCase	 =     collections.Counter(_UpperCAmelCase					) & collections.Counter(_UpperCAmelCase					)
			_UpperCAmelCase	 =     sum(common.values()					)
			if len(_UpperCAmelCase					) == 0 or len(_UpperCAmelCase					) == 0:
						# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
						return int(gold_toks == pred_toks					)
			if num_same == 0:
						return 0
			_UpperCAmelCase	 =     1.0 * num_same / len(_UpperCAmelCase					)
			_UpperCAmelCase	 =     1.0 * num_same / len(_UpperCAmelCase					)
			_UpperCAmelCase	 =     (2 * precision * recall) / (precision + recall)
			return fa





def 							A			(  _UpperCAmelCase       :     List[Any]							,     _UpperCAmelCase       :     List[str]					)       ->				Dict:







			'''simple docstring'''
			_UpperCAmelCase	 =     {}
			_UpperCAmelCase	 =     {}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												_UpperCAmelCase	 =     qa['id']
												_UpperCAmelCase	 =     [t for t in qa['answers']['text'] if normalize_answer(_UpperCAmelCase					)]
												if not gold_answers:
															# For unanswerable questions, only correct answer is empty string
															_UpperCAmelCase	 =     ['']
												if qid not in preds:
															print(F"Missing prediction for {qid}"					)
															continue
												_UpperCAmelCase	 =     preds[qid]
												# Take max over all gold answers
												_UpperCAmelCase	 =     max(compute_exact(_UpperCAmelCase							,     _UpperCAmelCase					) for a in gold_answers					)
												_UpperCAmelCase	 =     max(compute_fa(_UpperCAmelCase							,     _UpperCAmelCase					) for a in gold_answers					)
			return exact_scores, fa_scores





def 							A			(  _UpperCAmelCase       :     Optional[Any]							,     _UpperCAmelCase       :     Dict							,     _UpperCAmelCase       :     List[str]							,     _UpperCAmelCase       :     int					)       ->				Tuple:







			'''simple docstring'''
			_UpperCAmelCase	 =     {}
			for qid, s in scores.items():
						_UpperCAmelCase	 =     na_probs[qid] > na_prob_thresh
						if pred_na:
									_UpperCAmelCase	 =     float(not qid_to_has_ans[qid]					)
						else:
									_UpperCAmelCase	 =     s
			return new_scores





def 							A			(  _UpperCAmelCase       :     Dict							,     _UpperCAmelCase       :     int							,     _UpperCAmelCase       :     int=None					)       ->				Any:







			'''simple docstring'''
			if not qid_list:
						_UpperCAmelCase	 =     len(_UpperCAmelCase					)
						return collections.OrderedDict(
						    [
						        ('exact', 100.0 * sum(exact_scores.values()					) / total),
						        ('f1', 100.0 * sum(fa_scores.values()					) / total),
						        ('total', total),
						    ]					)
			else:
						_UpperCAmelCase	 =     len(_UpperCAmelCase					)
						return collections.OrderedDict(
						    [
						        ('exact', 100.0 * sum(exact_scores[k] for k in qid_list					) / total),
						        ('f1', 100.0 * sum(fa_scores[k] for k in qid_list					) / total),
						        ('total', total),
						    ]					)





def 							A			(  _UpperCAmelCase       :     List[Any]							,     _UpperCAmelCase       :     Any							,     _UpperCAmelCase       :     Tuple					)       ->				Optional[int]:







			'''simple docstring'''
			for k in new_eval:
						_UpperCAmelCase	 =     new_eval[k]





def 							A			(  _UpperCAmelCase       :     Optional[Any]							,     _UpperCAmelCase       :     int							,     _UpperCAmelCase       :     int							,     _UpperCAmelCase       :     Optional[Any]					)       ->				str:







			'''simple docstring'''
			plt.step(_UpperCAmelCase							,     _UpperCAmelCase							,     color='b'							,     alpha=0.2							,     where='post'					)
			plt.fill_between(_UpperCAmelCase							,     _UpperCAmelCase							,     step='post'							,     alpha=0.2							,     color='b'					)
			plt.xlabel('Recall'					)
			plt.ylabel('Precision'					)
			plt.xlim([0.0, 1.05]					)
			plt.ylim([0.0, 1.05]					)
			plt.title(_UpperCAmelCase					)
			plt.savefig(_UpperCAmelCase					)
			plt.clf()





def 							A			(  _UpperCAmelCase       :     Union[str, Any]							,     _UpperCAmelCase       :     Tuple							,     _UpperCAmelCase       :     Any							,     _UpperCAmelCase       :     Optional[Any]							,     _UpperCAmelCase       :     Optional[Any]=None							,     _UpperCAmelCase       :     Union[str, Any]=None					)       ->				Union[str, Any]:







			'''simple docstring'''
			_UpperCAmelCase	 =     sorted(_UpperCAmelCase							,     key=lambda _UpperCAmelCase					: na_probs[k]					)
			_UpperCAmelCase	 =     0.0
			_UpperCAmelCase	 =     1.0
			_UpperCAmelCase	 =     0.0
			_UpperCAmelCase	 =     [1.0]
			_UpperCAmelCase	 =     [0.0]
			_UpperCAmelCase	 =     0.0
			for i, qid in enumerate(_UpperCAmelCase					):
						if qid_to_has_ans[qid]:
									true_pos += scores[qid]
						_UpperCAmelCase	 =     true_pos / float(i + 1					)
						_UpperCAmelCase	 =     true_pos / float(_UpperCAmelCase					)
						if i == len(_UpperCAmelCase					) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
									# i.e., if we can put a threshold after this point
									avg_prec += cur_p * (cur_r - recalls[-1])
									precisions.append(_UpperCAmelCase					)
									recalls.append(_UpperCAmelCase					)
			if out_image:
						plot_pr_curve(_UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase					)
			return {"ap": 100.0 * avg_prec}





def 							A			(  _UpperCAmelCase       :     int							,     _UpperCAmelCase       :     Optional[int]							,     _UpperCAmelCase       :     Optional[Any]							,     _UpperCAmelCase       :     str							,     _UpperCAmelCase       :     Union[str, Any]							,     _UpperCAmelCase       :     Optional[Any]					)       ->				Dict:







			'''simple docstring'''
			if out_image_dir and not os.path.exists(_UpperCAmelCase					):
						os.makedirs(_UpperCAmelCase					)
			_UpperCAmelCase	 =     sum(1 for v in qid_to_has_ans.values() if v					)
			if num_true_pos == 0:
						return
			_UpperCAmelCase	 =     make_precision_recall_eval(
			    _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     out_image=os.path.join(_UpperCAmelCase							,     'pr_exact.png'					)							,     title='Precision-Recall curve for Exact Match score'							,     )
			_UpperCAmelCase	 =     make_precision_recall_eval(
			    _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     out_image=os.path.join(_UpperCAmelCase							,     'pr_f1.png'					)							,     title='Precision-Recall curve for F1 score'							,     )
			_UpperCAmelCase	 =     {k: float(_UpperCAmelCase					) for k, v in qid_to_has_ans.items()}
			_UpperCAmelCase	 =     make_precision_recall_eval(
			    _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     out_image=os.path.join(_UpperCAmelCase							,     'pr_oracle.png'					)							,     title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'							,     )
			merge_eval(_UpperCAmelCase							,     _UpperCAmelCase							,     'pr_exact'					)
			merge_eval(_UpperCAmelCase							,     _UpperCAmelCase							,     'pr_f1'					)
			merge_eval(_UpperCAmelCase							,     _UpperCAmelCase							,     'pr_oracle'					)





def 							A			(  _UpperCAmelCase       :     List[str]							,     _UpperCAmelCase       :     Optional[Any]							,     _UpperCAmelCase       :     List[str]							,     _UpperCAmelCase       :     Any					)       ->				Any:







			'''simple docstring'''
			if not qid_list:
						return
			_UpperCAmelCase	 =     [na_probs[k] for k in qid_list]
			_UpperCAmelCase	 =     np.ones_like(_UpperCAmelCase					) / float(len(_UpperCAmelCase					)					)
			plt.hist(_UpperCAmelCase							,     weights=_UpperCAmelCase							,     bins=20							,     range=(0.0, 1.0)					)
			plt.xlabel('Model probability of no-answer'					)
			plt.ylabel('Proportion of dataset'					)
			plt.title(F"Histogram of no-answer probability: {name}"					)
			plt.savefig(os.path.join(_UpperCAmelCase							,     F"na_prob_hist_{name}.png"					)					)
			plt.clf()





def 							A			(  _UpperCAmelCase       :     Optional[Any]							,     _UpperCAmelCase       :     int							,     _UpperCAmelCase       :     Optional[int]							,     _UpperCAmelCase       :     str					)       ->				Any:







			'''simple docstring'''
			_UpperCAmelCase	 =     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]					)
			_UpperCAmelCase	 =     num_no_ans
			_UpperCAmelCase	 =     cur_score
			_UpperCAmelCase	 =     0.0
			_UpperCAmelCase	 =     sorted(_UpperCAmelCase							,     key=lambda _UpperCAmelCase					: na_probs[k]					)
			for i, qid in enumerate(_UpperCAmelCase					):
						if qid not in scores:
									continue
						if qid_to_has_ans[qid]:
									_UpperCAmelCase	 =     scores[qid]
						else:
									if preds[qid]:
												_UpperCAmelCase	 =     -1
									else:
												_UpperCAmelCase	 =     0
						cur_score += diff
						if cur_score > best_score:
									_UpperCAmelCase	 =     cur_score
									_UpperCAmelCase	 =     na_probs[qid]
			return 100.0 * best_score / len(_UpperCAmelCase					), best_thresh





def 							A			(  _UpperCAmelCase       :     Optional[Any]							,     _UpperCAmelCase       :     Any							,     _UpperCAmelCase       :     Tuple							,     _UpperCAmelCase       :     int							,     _UpperCAmelCase       :     Optional[int]							,     _UpperCAmelCase       :     Union[str, Any]					)       ->				List[Any]:







			'''simple docstring'''
			_UpperCAmelCase ,  _UpperCAmelCase	 =     find_best_thresh(_UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase					)
			_UpperCAmelCase ,  _UpperCAmelCase	 =     find_best_thresh(_UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase					)
			_UpperCAmelCase	 =     best_exact
			_UpperCAmelCase	 =     exact_thresh
			_UpperCAmelCase	 =     best_fa
			_UpperCAmelCase	 =     fa_thresh





def 							A			(  )       ->				Optional[Any]:







			'''simple docstring'''
			with open(OPTS.data_file					) as f:
						_UpperCAmelCase	 =     json.load(_UpperCAmelCase					)
						_UpperCAmelCase	 =     dataset_json['data']
			with open(OPTS.pred_file					) as f:
						_UpperCAmelCase	 =     json.load(_UpperCAmelCase					)
			if OPTS.na_prob_file:
						with open(OPTS.na_prob_file					) as f:
									_UpperCAmelCase	 =     json.load(_UpperCAmelCase					)
			else:
						_UpperCAmelCase	 =     {k: 0.0 for k in preds}
			_UpperCAmelCase	 =     make_qid_to_has_ans(_UpperCAmelCase					)  # maps qid to True/False
			_UpperCAmelCase	 =     [k for k, v in qid_to_has_ans.items() if v]
			_UpperCAmelCase	 =     [k for k, v in qid_to_has_ans.items() if not v]
			_UpperCAmelCase ,  _UpperCAmelCase	 =     get_raw_scores(_UpperCAmelCase							,     _UpperCAmelCase					)
			_UpperCAmelCase	 =     apply_no_ans_threshold(_UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     OPTS.na_prob_thresh					)
			_UpperCAmelCase	 =     apply_no_ans_threshold(_UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     OPTS.na_prob_thresh					)
			_UpperCAmelCase	 =     make_eval_dict(_UpperCAmelCase							,     _UpperCAmelCase					)
			if has_ans_qids:
						_UpperCAmelCase	 =     make_eval_dict(_UpperCAmelCase							,     _UpperCAmelCase							,     qid_list=_UpperCAmelCase					)
						merge_eval(_UpperCAmelCase							,     _UpperCAmelCase							,     'HasAns'					)
			if no_ans_qids:
						_UpperCAmelCase	 =     make_eval_dict(_UpperCAmelCase							,     _UpperCAmelCase							,     qid_list=_UpperCAmelCase					)
						merge_eval(_UpperCAmelCase							,     _UpperCAmelCase							,     'NoAns'					)
			if OPTS.na_prob_file:
						find_all_best_thresh(_UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase					)
			if OPTS.na_prob_file and OPTS.out_image_dir:
						run_precision_recall_analysis(_UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase							,     OPTS.out_image_dir					)
						histogram_na_prob(_UpperCAmelCase							,     _UpperCAmelCase							,     OPTS.out_image_dir							,     'hasAns'					)
						histogram_na_prob(_UpperCAmelCase							,     _UpperCAmelCase							,     OPTS.out_image_dir							,     'noAns'					)
			if OPTS.out_file:
						with open(OPTS.out_file							,     'w'					) as f:
									json.dump(_UpperCAmelCase							,     _UpperCAmelCase					)
			else:
						print(json.dumps(_UpperCAmelCase							,     indent=2					)					)


if __name__ == "__main__":
							UpperCAmelCase__     = parse_args()
							if OPTS.out_image_dir:
														import matplotlib

														matplotlib.use("Agg")
														import matplotlib.pyplot as plt
							main()





import unittest

from knapsack import knapsack as k







class     __lowerCAmelCase					(    unittest.TestCase		):



			def 					_lowerCamelCase (		self     :						Optional[Any])     ->					Any:



						"""simple docstring"""






						_UpperCAmelCase	 =     0
						_UpperCAmelCase	 =     [0]
						_UpperCAmelCase	 =     [0]
						_UpperCAmelCase	 =     len(A)
						self.assertEqual(k.knapsack(A				,	A				,	A				,	A)				,	0)

						_UpperCAmelCase	 =     [60]
						_UpperCAmelCase	 =     [10]
						_UpperCAmelCase	 =     len(A)
						self.assertEqual(k.knapsack(A				,	A				,	A				,	A)				,	0)



			def 					_lowerCamelCase (		self     :						str)     ->					List[str]:



						"""simple docstring"""






						_UpperCAmelCase	 =     3
						_UpperCAmelCase	 =     [1, 2, 3]
						_UpperCAmelCase	 =     [3, 2, 1]
						_UpperCAmelCase	 =     len(A)
						self.assertEqual(k.knapsack(A				,	A				,	A				,	A)				,	5)



			def 					_lowerCamelCase (		self     :						Tuple)     ->					Tuple:



						"""simple docstring"""






						_UpperCAmelCase	 =     50
						_UpperCAmelCase	 =     [60, 1_00, 1_20]
						_UpperCAmelCase	 =     [10, 20, 30]
						_UpperCAmelCase	 =     len(A)
						self.assertEqual(k.knapsack(A				,	A				,	A				,	A)				,	2_20)


if __name__ == "__main__":
							unittest.main()
