



import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


snake_case__					:     List[str]    		=   re.compile(R"""\b(a|an|the)\b""", re.UNICODE)

snake_case__					:     List[Any]    		=   None



def    snake_case_				(    ):
	__lowercase		       =			argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."   )
	parser.add_argument("data_file"  ,	metavar="data.json"  ,	help="Input data JSON file."   )
	parser.add_argument("pred_file"  ,	metavar="pred.json"  ,	help="Model predictions."   )
	parser.add_argument(
	    "--out-file"  ,	"-o"  ,	metavar="eval.json"  ,	help="Write accuracy metrics to file (default is stdout)."   )
	parser.add_argument(
	    "--na-prob-file"  ,	"-n"  ,	metavar="na_prob.json"  ,	help="Model estimates of probability of no answer."   )
	parser.add_argument(
	    "--na-prob-thresh"  ,	"-t"  ,	type=_SCREAMING_SNAKE_CASE  ,	default=1.0  ,	help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."  ,	)
	parser.add_argument(
	    "--out-image-dir"  ,	"-p"  ,	metavar="out_images"  ,	default=_SCREAMING_SNAKE_CASE  ,	help="Save precision-recall curves to directory."   )
	parser.add_argument("--verbose"  ,	"-v"  ,	action="store_true"   )
	if len(sys.argv   ) == 1:
		parser.print_help()
		sys.exit(1   )
	return parser.parse_args()



def    snake_case_				(    _SCREAMING_SNAKE_CASE   ):
	__lowercase		       =			{}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				__lowercase		       =			bool(qa["answers"]["text"]   )
	return qid_to_has_ans



def    snake_case_				(    _SCREAMING_SNAKE_CASE   ):

	def remove_articles(_SCREAMING_SNAKE_CASE   ):
		return ARTICLES_REGEX.sub(" "  ,	_SCREAMING_SNAKE_CASE   )

	def white_space_fix(_SCREAMING_SNAKE_CASE   ):
		return " ".join(text.split()   )

	def remove_punc(_SCREAMING_SNAKE_CASE   ):
		__lowercase		       =			set(string.punctuation   )
		return "".join(ch for ch in text if ch not in exclude   )

	def lower(_SCREAMING_SNAKE_CASE   ):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(_SCREAMING_SNAKE_CASE   )   )   )   )



def    snake_case_				(    _SCREAMING_SNAKE_CASE   ):
	if not s:
		return []
	return normalize_answer(_SCREAMING_SNAKE_CASE   ).split()



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ):
	return int(normalize_answer(_SCREAMING_SNAKE_CASE   ) == normalize_answer(_SCREAMING_SNAKE_CASE   )   )



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ):
	__lowercase		       =			get_tokens(_SCREAMING_SNAKE_CASE   )
	__lowercase		       =			get_tokens(_SCREAMING_SNAKE_CASE   )
	__lowercase		       =			collections.Counter(_SCREAMING_SNAKE_CASE   ) & collections.Counter(_SCREAMING_SNAKE_CASE   )
	__lowercase		       =			sum(common.values()   )
	if len(_SCREAMING_SNAKE_CASE   ) == 0 or len(_SCREAMING_SNAKE_CASE   ) == 0:
		# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
		return int(gold_toks == pred_toks   )
	if num_same == 0:
		return 0
	__lowercase		       =			1.0 * num_same / len(_SCREAMING_SNAKE_CASE   )
	__lowercase		       =			1.0 * num_same / len(_SCREAMING_SNAKE_CASE   )
	__lowercase		       =			(2 * precision * recall) / (precision + recall)
	return fa



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ):
	__lowercase		       =			{}
	__lowercase		       =			{}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				__lowercase		       =			qa["""id"""]
				__lowercase		       =			[t for t in qa["""answers"""]["""text"""] if normalize_answer(_SCREAMING_SNAKE_CASE   )]
				if not gold_answers:
					# For unanswerable questions, only correct answer is empty string
					__lowercase		       =			[""""""]
				if qid not in preds:
					print(F"""Missing prediction for {qid}"""   )
					continue
				__lowercase		       =			preds[qid]
				# Take max over all gold answers
				__lowercase		       =			max(compute_exact(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ) for a in gold_answers   )
				__lowercase		       =			max(compute_fa(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ) for a in gold_answers   )
	return exact_scores, fa_scores



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ):
	__lowercase		       =			{}
	for qid, s in scores.items():
		__lowercase		       =			na_probs[qid] > na_prob_thresh
		if pred_na:
			__lowercase		       =			float(not qid_to_has_ans[qid]   )
		else:
			__lowercase		       =			s
	return new_scores



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE=None   ):
	if not qid_list:
		__lowercase		       =			len(_SCREAMING_SNAKE_CASE   )
		return collections.OrderedDict(
		    [
		        ("exact", 1_0_0.0 * sum(exact_scores.values()   ) / total),
		        ("f1", 1_0_0.0 * sum(fa_scores.values()   ) / total),
		        ("total", total),
		    ]   )
	else:
		__lowercase		       =			len(_SCREAMING_SNAKE_CASE   )
		return collections.OrderedDict(
		    [
		        ("exact", 1_0_0.0 * sum(exact_scores[k] for k in qid_list   ) / total),
		        ("f1", 1_0_0.0 * sum(fa_scores[k] for k in qid_list   ) / total),
		        ("total", total),
		    ]   )



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ):
	for k in new_eval:
		__lowercase		       =			new_eval[k]



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ):
	plt.step(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	color="b"  ,	alpha=0.2  ,	where="post"   )
	plt.fill_between(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	step="post"  ,	alpha=0.2  ,	color="b"   )
	plt.xlabel("Recall"   )
	plt.ylabel("Precision"   )
	plt.xlim([0.0, 1.0_5]   )
	plt.ylim([0.0, 1.0_5]   )
	plt.title(_SCREAMING_SNAKE_CASE   )
	plt.savefig(_SCREAMING_SNAKE_CASE   )
	plt.clf()



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE=None  ,	_SCREAMING_SNAKE_CASE=None   ):
	__lowercase		       =			sorted(_SCREAMING_SNAKE_CASE  ,	key=lambda _SCREAMING_SNAKE_CASE   : na_probs[k]   )
	__lowercase		       =			0.0
	__lowercase		       =			1.0
	__lowercase		       =			0.0
	__lowercase		       =			[1.0]
	__lowercase		       =			[0.0]
	__lowercase		       =			0.0
	for i, qid in enumerate(_SCREAMING_SNAKE_CASE   ):
		if qid_to_has_ans[qid]:
			true_pos += scores[qid]
		__lowercase		       =			true_pos / float(i + 1   )
		__lowercase		       =			true_pos / float(_SCREAMING_SNAKE_CASE   )
		if i == len(_SCREAMING_SNAKE_CASE   ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
			# i.e., if we can put a threshold after this point
			avg_prec += cur_p * (cur_r - recalls[-1])
			precisions.append(_SCREAMING_SNAKE_CASE   )
			recalls.append(_SCREAMING_SNAKE_CASE   )
	if out_image:
		plot_pr_curve(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   )
	return {"ap": 1_0_0.0 * avg_prec}



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ):
	if out_image_dir and not os.path.exists(_SCREAMING_SNAKE_CASE   ):
		os.makedirs(_SCREAMING_SNAKE_CASE   )
	__lowercase		       =			sum(1 for v in qid_to_has_ans.values() if v   )
	if num_true_pos == 0:
		return
	__lowercase		       =			make_precision_recall_eval(
	    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	out_image=os.path.join(_SCREAMING_SNAKE_CASE  ,	"pr_exact.png"   )  ,	title="Precision-Recall curve for Exact Match score"  ,	)
	__lowercase		       =			make_precision_recall_eval(
	    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	out_image=os.path.join(_SCREAMING_SNAKE_CASE  ,	"pr_f1.png"   )  ,	title="Precision-Recall curve for F1 score"  ,	)
	__lowercase		       =			{k: float(_SCREAMING_SNAKE_CASE   ) for k, v in qid_to_has_ans.items()}
	__lowercase		       =			make_precision_recall_eval(
	    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	out_image=os.path.join(_SCREAMING_SNAKE_CASE  ,	"pr_oracle.png"   )  ,	title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"  ,	)
	merge_eval(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	"pr_exact"   )
	merge_eval(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	"pr_f1"   )
	merge_eval(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	"pr_oracle"   )



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ):
	if not qid_list:
		return
	__lowercase		       =			[na_probs[k] for k in qid_list]
	__lowercase		       =			np.ones_like(_SCREAMING_SNAKE_CASE   ) / float(len(_SCREAMING_SNAKE_CASE   )   )
	plt.hist(_SCREAMING_SNAKE_CASE  ,	weights=_SCREAMING_SNAKE_CASE  ,	bins=2_0  ,	range=(0.0, 1.0)   )
	plt.xlabel("Model probability of no-answer"   )
	plt.ylabel("Proportion of dataset"   )
	plt.title(F"""Histogram of no-answer probability: {name}"""   )
	plt.savefig(os.path.join(_SCREAMING_SNAKE_CASE  ,	F"""na_prob_hist_{name}.png"""   )   )
	plt.clf()



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ):
	__lowercase		       =			sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]   )
	__lowercase		       =			num_no_ans
	__lowercase		       =			cur_score
	__lowercase		       =			0.0
	__lowercase		       =			sorted(_SCREAMING_SNAKE_CASE  ,	key=lambda _SCREAMING_SNAKE_CASE   : na_probs[k]   )
	for i, qid in enumerate(_SCREAMING_SNAKE_CASE   ):
		if qid not in scores:
			continue
		if qid_to_has_ans[qid]:
			__lowercase		       =			scores[qid]
		else:
			if preds[qid]:
				__lowercase		       =			-1
			else:
				__lowercase		       =			0
		cur_score += diff
		if cur_score > best_score:
			__lowercase		       =			cur_score
			__lowercase		       =			na_probs[qid]
	return 1_0_0.0 * best_score / len(_SCREAMING_SNAKE_CASE   ), best_thresh



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ):
	__lowercase		       =			find_best_thresh(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   )
	__lowercase		       =			find_best_thresh(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   )
	__lowercase		       =			best_exact
	__lowercase		       =			exact_thresh
	__lowercase		       =			best_fa
	__lowercase		       =			fa_thresh



def    snake_case_				(    ):
	with open(OPTS.data_file   ) as f:
		__lowercase		       =			json.load(_SCREAMING_SNAKE_CASE   )
		__lowercase		       =			dataset_json["""data"""]
	with open(OPTS.pred_file   ) as f:
		__lowercase		       =			json.load(_SCREAMING_SNAKE_CASE   )
	if OPTS.na_prob_file:
		with open(OPTS.na_prob_file   ) as f:
			__lowercase		       =			json.load(_SCREAMING_SNAKE_CASE   )
	else:
		__lowercase		       =			{k: 0.0 for k in preds}
	__lowercase		       =			make_qid_to_has_ans(_SCREAMING_SNAKE_CASE   )  # maps qid to True/False
	__lowercase		       =			[k for k, v in qid_to_has_ans.items() if v]
	__lowercase		       =			[k for k, v in qid_to_has_ans.items() if not v]
	__lowercase		       =			get_raw_scores(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   )
	__lowercase		       =			apply_no_ans_threshold(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	OPTS.na_prob_thresh   )
	__lowercase		       =			apply_no_ans_threshold(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	OPTS.na_prob_thresh   )
	__lowercase		       =			make_eval_dict(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   )
	if has_ans_qids:
		__lowercase		       =			make_eval_dict(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	qid_list=_SCREAMING_SNAKE_CASE   )
		merge_eval(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	"HasAns"   )
	if no_ans_qids:
		__lowercase		       =			make_eval_dict(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	qid_list=_SCREAMING_SNAKE_CASE   )
		merge_eval(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	"NoAns"   )
	if OPTS.na_prob_file:
		find_all_best_thresh(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   )
	if OPTS.na_prob_file and OPTS.out_image_dir:
		run_precision_recall_analysis(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	OPTS.out_image_dir   )
		histogram_na_prob(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	OPTS.out_image_dir  ,	"hasAns"   )
		histogram_na_prob(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	OPTS.out_image_dir  ,	"noAns"   )
	if OPTS.out_file:
		with open(OPTS.out_file  ,	"w"   ) as f:
			json.dump(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   )
	else:
		print(json.dumps(_SCREAMING_SNAKE_CASE  ,	indent=2   )   )


if __name__ == "__main__":
	snake_case__					:     Optional[int]    		=   parse_args()
	if OPTS.out_image_dir:
		import matplotlib

		matplotlib.use("""Agg""")
		import matplotlib.pyplot as plt
	main()








import numpy as np

snake_case__					:     Tuple    		=   [
    ["""a""", """b""", """c""", """d""", """e"""],
    ["""f""", """g""", """h""", """i""", """k"""],
    ["""l""", """m""", """n""", """o""", """p"""],
    ["""q""", """r""", """s""", """t""", """u"""],
    ["""v""", """w""", """x""", """y""", """z"""],
]

class   _A   :


						'''simple docstring'''





						def __init__(	self	:							Dict	):






							'''simple docstring'''



							__lowercase		       =			np.array(lowerCamelCase	)



						def     _snake_case	(	self	:							Union[str, Any]							,      lowerCamelCase	:							str	):






							'''simple docstring'''



							__lowercase       ,    __lowercase		       =			np.where(letter == self.SQUARE	)
							__lowercase		       =			np.concatenate([indexa + 1, indexa + 1]	)
							return indexes



						def     _snake_case	(	self	:							List[Any]							,      lowerCamelCase	:							int							,      lowerCamelCase	:							int	):






							'''simple docstring'''



							__lowercase		       =			self.SQUARE[indexa - 1, indexa - 1]
							return letter



						def     _snake_case	(	self	:							int							,      lowerCamelCase	:							str	):






							'''simple docstring'''



							__lowercase		       =			message.lower()
							__lowercase		       =			message.replace(" "							,      ""	)
							__lowercase		       =			message.replace("j"							,      "i"	)

							__lowercase		       =			np.empty((2, len(lowerCamelCase	))	)
							for letter_index in range(len(lowerCamelCase	)	):
								__lowercase		       =			self.letter_to_numbers(message[letter_index]	)

								__lowercase		       =			numbers[0]
								__lowercase		       =			numbers[1]

							__lowercase		       =			first_step.reshape(2 * len(lowerCamelCase	)	)
							__lowercase		       =			""
							for numbers_index in range(len(lowerCamelCase	)	):
								__lowercase		       =			int(second_step[numbers_index * 2]	)
								__lowercase		       =			int(second_step[(numbers_index * 2) + 1]	)
								__lowercase		       =			self.numbers_to_letter(lowerCamelCase							,      lowerCamelCase	)
								__lowercase		       =			encoded_message + letter

							return encoded_message



						def     _snake_case	(	self	:							Optional[Any]							,      lowerCamelCase	:							str	):






							'''simple docstring'''



							__lowercase		       =			message.lower()
							message.replace(" "							,      ""	)
							__lowercase		       =			np.empty(2 * len(lowerCamelCase	)	)
							for letter_index in range(len(lowerCamelCase	)	):
								__lowercase		       =			self.letter_to_numbers(message[letter_index]	)
								__lowercase		       =			numbers[0]
								__lowercase		       =			numbers[1]

							__lowercase		       =			first_step.reshape((2, len(lowerCamelCase	))	)
							__lowercase		       =			""
							for numbers_index in range(len(lowerCamelCase	)	):
								__lowercase		       =			int(second_step[0, numbers_index]	)
								__lowercase		       =			int(second_step[1, numbers_index]	)
								__lowercase		       =			self.numbers_to_letter(lowerCamelCase							,      lowerCamelCase	)
								__lowercase		       =			decoded_message + letter

							return decoded_message



