



"""simple docstring"""
import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__SCREAMING_SNAKE_CASE  :						List[Any]    							=      re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

__SCREAMING_SNAKE_CASE  :						Optional[Any]    							=      None



def        lowerCAmelCase_(   )	->       List[Any]:
				_lowerCamelCase        =				argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''							)
				parser.add_argument('''data_file'''						,   metavar='''data.json'''						,   help='''Input data JSON file.'''							)
				parser.add_argument('''pred_file'''						,   metavar='''pred.json'''						,   help='''Model predictions.'''							)
				parser.add_argument(
				    '''--out-file'''						,   '''-o'''						,   metavar='''eval.json'''						,   help='''Write accuracy metrics to file (default is stdout).'''							)
				parser.add_argument(
				    '''--na-prob-file'''						,   '''-n'''						,   metavar='''na_prob.json'''						,   help='''Model estimates of probability of no answer.'''							)
				parser.add_argument(
				    '''--na-prob-thresh'''						,   '''-t'''						,   type=lowercase_						,   default=1.0						,   help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''						,   )
				parser.add_argument(
				    '''--out-image-dir'''						,   '''-p'''						,   metavar='''out_images'''						,   default=lowercase_						,   help='''Save precision-recall curves to directory.'''							)
				parser.add_argument('''--verbose'''						,   '''-v'''						,   action='''store_true'''							)
				if len(sys.argv							) == 1:
								parser.print_help()
								sys.exit(1							)
				return parser.parse_args()



def        lowerCAmelCase_(   lowercase_     :							int							)	->       List[str]:
				_lowerCamelCase        =				{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																_lowerCamelCase        =				bool(qa['''answers''']['''text''']							)
				return qid_to_has_ans



def        lowerCAmelCase_(   lowercase_     :							str							)	->       List[str]:

				def remove_articles(lowercase_     :							List[Any]							):
								return ARTICLES_REGEX.sub(''' '''						,   lowercase_							)

				def white_space_fix(lowercase_     :							int							):
								return " ".join(text.split()							)

				def remove_punc(lowercase_     :							List[str]							):
								_lowerCamelCase        =				set(string.punctuation							)
								return "".join(ch for ch in text if ch not in exclude							)

				def lower(lowercase_     :							Tuple							):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(lowercase_							)							)							)							)



def        lowerCAmelCase_(   lowercase_     :							int							)	->       List[str]:
				if not s:
								return []
				return normalize_answer(lowercase_							).split()



def        lowerCAmelCase_(   lowercase_     :							Dict						,   lowercase_     :							Optional[Any]							)	->       Any:
				return int(normalize_answer(lowercase_							) == normalize_answer(lowercase_							)							)



def        lowerCAmelCase_(   lowercase_     :							Union[str, Any]						,   lowercase_     :							Any							)	->       List[str]:
				_lowerCamelCase        =				get_tokens(lowercase_							)
				_lowerCamelCase        =				get_tokens(lowercase_							)
				_lowerCamelCase        =				collections.Counter(lowercase_							) & collections.Counter(lowercase_							)
				_lowerCamelCase        =				sum(common.values()							)
				if len(lowercase_							) == 0 or len(lowercase_							) == 0:
								# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
								return int(gold_toks == pred_toks							)
				if num_same == 0:
								return 0
				_lowerCamelCase        =				1.0 * num_same / len(lowercase_							)
				_lowerCamelCase        =				1.0 * num_same / len(lowercase_							)
				_lowerCamelCase        =				(2 * precision * recall) / (precision + recall)
				return fa



def        lowerCAmelCase_(   lowercase_     :							int						,   lowercase_     :							Any							)	->       List[str]:
				_lowerCamelCase        =				{}
				_lowerCamelCase        =				{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																_lowerCamelCase        =				qa['''id''']
																_lowerCamelCase        =				[t for t in qa['''answers''']['''text'''] if normalize_answer(lowercase_							)]
																if not gold_answers:
																				# For unanswerable questions, only correct answer is empty string
																				_lowerCamelCase        =				['''''']
																if qid not in preds:
																				print(F"""Missing prediction for {qid}"""							)
																				continue
																_lowerCamelCase        =				preds[qid]
																# Take max over all gold answers
																_lowerCamelCase        =				max(compute_exact(lowercase_						,   lowercase_							) for a in gold_answers							)
																_lowerCamelCase        =				max(compute_fa(lowercase_						,   lowercase_							) for a in gold_answers							)
				return exact_scores, fa_scores



def        lowerCAmelCase_(   lowercase_     :							Optional[int]						,   lowercase_     :							List[str]						,   lowercase_     :							List[Any]						,   lowercase_     :							Optional[int]							)	->       Union[str, Any]:
				_lowerCamelCase        =				{}
				for qid, s in scores.items():
								_lowerCamelCase        =				na_probs[qid] > na_prob_thresh
								if pred_na:
												_lowerCamelCase        =				float(not qid_to_has_ans[qid]							)
								else:
												_lowerCamelCase        =				s
				return new_scores



def        lowerCAmelCase_(   lowercase_     :							str						,   lowercase_     :							Optional[int]						,   lowercase_     :							List[Any]=None							)	->       Any:
				if not qid_list:
								_lowerCamelCase        =				len(lowercase_							)
								return collections.OrderedDict(
								    [
								        ('''exact''', 1_0_0.0 * sum(exact_scores.values()							) / total),
								        ('''f1''', 1_0_0.0 * sum(fa_scores.values()							) / total),
								        ('''total''', total),
								    ]							)
				else:
								_lowerCamelCase        =				len(lowercase_							)
								return collections.OrderedDict(
								    [
								        ('''exact''', 1_0_0.0 * sum(exact_scores[k] for k in qid_list							) / total),
								        ('''f1''', 1_0_0.0 * sum(fa_scores[k] for k in qid_list							) / total),
								        ('''total''', total),
								    ]							)



def        lowerCAmelCase_(   lowercase_     :							Tuple						,   lowercase_     :							Tuple						,   lowercase_     :							Any							)	->       List[Any]:
				for k in new_eval:
								_lowerCamelCase        =				new_eval[k]



def        lowerCAmelCase_(   lowercase_     :							int						,   lowercase_     :							Tuple						,   lowercase_     :							Tuple						,   lowercase_     :							List[Any]							)	->       Optional[Any]:
				plt.step(lowercase_						,   lowercase_						,   color='''b'''						,   alpha=0.2						,   where='''post'''							)
				plt.fill_between(lowercase_						,   lowercase_						,   step='''post'''						,   alpha=0.2						,   color='''b'''							)
				plt.xlabel('''Recall'''							)
				plt.ylabel('''Precision'''							)
				plt.xlim([0.0, 1.0_5]							)
				plt.ylim([0.0, 1.0_5]							)
				plt.title(lowercase_							)
				plt.savefig(lowercase_							)
				plt.clf()



def        lowerCAmelCase_(   lowercase_     :							Dict						,   lowercase_     :							int						,   lowercase_     :							Union[str, Any]						,   lowercase_     :							Optional[int]						,   lowercase_     :							Union[str, Any]=None						,   lowercase_     :							Any=None							)	->       Union[str, Any]:
				_lowerCamelCase        =				sorted(lowercase_						,   key=lambda lowercase_							: na_probs[k]							)
				_lowerCamelCase        =				0.0
				_lowerCamelCase        =				1.0
				_lowerCamelCase        =				0.0
				_lowerCamelCase        =				[1.0]
				_lowerCamelCase        =				[0.0]
				_lowerCamelCase        =				0.0
				for i, qid in enumerate(lowercase_							):
								if qid_to_has_ans[qid]:
												true_pos += scores[qid]
								_lowerCamelCase        =				true_pos / float(i + 1							)
								_lowerCamelCase        =				true_pos / float(lowercase_							)
								if i == len(lowercase_							) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
												# i.e., if we can put a threshold after this point
												avg_prec += cur_p * (cur_r - recalls[-1])
												precisions.append(lowercase_							)
												recalls.append(lowercase_							)
				if out_image:
								plot_pr_curve(lowercase_						,   lowercase_						,   lowercase_						,   lowercase_							)
				return {"ap": 1_0_0.0 * avg_prec}



def        lowerCAmelCase_(   lowercase_     :							int						,   lowercase_     :							Optional[int]						,   lowercase_     :							Any						,   lowercase_     :							Optional[int]						,   lowercase_     :							Dict						,   lowercase_     :							Union[str, Any]							)	->       str:
				if out_image_dir and not os.path.exists(lowercase_							):
								os.makedirs(lowercase_							)
				_lowerCamelCase        =				sum(1 for v in qid_to_has_ans.values() if v							)
				if num_true_pos == 0:
								return
				_lowerCamelCase        =				make_precision_recall_eval(
				    lowercase_						,   lowercase_						,   lowercase_						,   lowercase_						,   out_image=os.path.join(lowercase_						,   '''pr_exact.png'''							)						,   title='''Precision-Recall curve for Exact Match score'''						,   )
				_lowerCamelCase        =				make_precision_recall_eval(
				    lowercase_						,   lowercase_						,   lowercase_						,   lowercase_						,   out_image=os.path.join(lowercase_						,   '''pr_f1.png'''							)						,   title='''Precision-Recall curve for F1 score'''						,   )
				_lowerCamelCase        =				{k: float(lowercase_							) for k, v in qid_to_has_ans.items()}
				_lowerCamelCase        =				make_precision_recall_eval(
				    lowercase_						,   lowercase_						,   lowercase_						,   lowercase_						,   out_image=os.path.join(lowercase_						,   '''pr_oracle.png'''							)						,   title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''						,   )
				merge_eval(lowercase_						,   lowercase_						,   '''pr_exact'''							)
				merge_eval(lowercase_						,   lowercase_						,   '''pr_f1'''							)
				merge_eval(lowercase_						,   lowercase_						,   '''pr_oracle'''							)



def        lowerCAmelCase_(   lowercase_     :							Dict						,   lowercase_     :							Dict						,   lowercase_     :							Union[str, Any]						,   lowercase_     :							List[Any]							)	->       Tuple:
				if not qid_list:
								return
				_lowerCamelCase        =				[na_probs[k] for k in qid_list]
				_lowerCamelCase        =				np.ones_like(lowercase_							) / float(len(lowercase_							)							)
				plt.hist(lowercase_						,   weights=lowercase_						,   bins=20						,   range=(0.0, 1.0)							)
				plt.xlabel('''Model probability of no-answer'''							)
				plt.ylabel('''Proportion of dataset'''							)
				plt.title(F"""Histogram of no-answer probability: {name}"""							)
				plt.savefig(os.path.join(lowercase_						,   F"""na_prob_hist_{name}.png"""							)							)
				plt.clf()



def        lowerCAmelCase_(   lowercase_     :							Tuple						,   lowercase_     :							Dict						,   lowercase_     :							Dict						,   lowercase_     :							List[Any]							)	->       Optional[Any]:
				_lowerCamelCase        =				sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]							)
				_lowerCamelCase        =				num_no_ans
				_lowerCamelCase        =				cur_score
				_lowerCamelCase        =				0.0
				_lowerCamelCase        =				sorted(lowercase_						,   key=lambda lowercase_							: na_probs[k]							)
				for i, qid in enumerate(lowercase_							):
								if qid not in scores:
												continue
								if qid_to_has_ans[qid]:
												_lowerCamelCase        =				scores[qid]
								else:
												if preds[qid]:
																_lowerCamelCase        =				-1
												else:
																_lowerCamelCase        =				0
								cur_score += diff
								if cur_score > best_score:
												_lowerCamelCase        =				cur_score
												_lowerCamelCase        =				na_probs[qid]
				return 1_0_0.0 * best_score / len(lowercase_							), best_thresh



def        lowerCAmelCase_(   lowercase_     :							Tuple						,   lowercase_     :							Tuple						,   lowercase_     :							Optional[Any]						,   lowercase_     :							Dict						,   lowercase_     :							Tuple						,   lowercase_     :							Optional[Any]							)	->       Dict:
				_lowerCamelCase  ,     _lowerCamelCase        =				find_best_thresh(lowercase_						,   lowercase_						,   lowercase_						,   lowercase_							)
				_lowerCamelCase  ,     _lowerCamelCase        =				find_best_thresh(lowercase_						,   lowercase_						,   lowercase_						,   lowercase_							)
				_lowerCamelCase        =				best_exact
				_lowerCamelCase        =				exact_thresh
				_lowerCamelCase        =				best_fa
				_lowerCamelCase        =				fa_thresh



def        lowerCAmelCase_(   )	->       Tuple:
				with open(OPTS.data_file							) as f:
								_lowerCamelCase        =				json.load(lowercase_							)
								_lowerCamelCase        =				dataset_json['''data''']
				with open(OPTS.pred_file							) as f:
								_lowerCamelCase        =				json.load(lowercase_							)
				if OPTS.na_prob_file:
								with open(OPTS.na_prob_file							) as f:
												_lowerCamelCase        =				json.load(lowercase_							)
				else:
								_lowerCamelCase        =				{k: 0.0 for k in preds}
				_lowerCamelCase        =				make_qid_to_has_ans(lowercase_							)  # maps qid to True/False
				_lowerCamelCase        =				[k for k, v in qid_to_has_ans.items() if v]
				_lowerCamelCase        =				[k for k, v in qid_to_has_ans.items() if not v]
				_lowerCamelCase  ,     _lowerCamelCase        =				get_raw_scores(lowercase_						,   lowercase_							)
				_lowerCamelCase        =				apply_no_ans_threshold(lowercase_						,   lowercase_						,   lowercase_						,   OPTS.na_prob_thresh							)
				_lowerCamelCase        =				apply_no_ans_threshold(lowercase_						,   lowercase_						,   lowercase_						,   OPTS.na_prob_thresh							)
				_lowerCamelCase        =				make_eval_dict(lowercase_						,   lowercase_							)
				if has_ans_qids:
								_lowerCamelCase        =				make_eval_dict(lowercase_						,   lowercase_						,   qid_list=lowercase_							)
								merge_eval(lowercase_						,   lowercase_						,   '''HasAns'''							)
				if no_ans_qids:
								_lowerCamelCase        =				make_eval_dict(lowercase_						,   lowercase_						,   qid_list=lowercase_							)
								merge_eval(lowercase_						,   lowercase_						,   '''NoAns'''							)
				if OPTS.na_prob_file:
								find_all_best_thresh(lowercase_						,   lowercase_						,   lowercase_						,   lowercase_						,   lowercase_						,   lowercase_							)
				if OPTS.na_prob_file and OPTS.out_image_dir:
								run_precision_recall_analysis(lowercase_						,   lowercase_						,   lowercase_						,   lowercase_						,   lowercase_						,   OPTS.out_image_dir							)
								histogram_na_prob(lowercase_						,   lowercase_						,   OPTS.out_image_dir						,   '''hasAns'''							)
								histogram_na_prob(lowercase_						,   lowercase_						,   OPTS.out_image_dir						,   '''noAns'''							)
				if OPTS.out_file:
								with open(OPTS.out_file						,   '''w'''							) as f:
												json.dump(lowercase_						,   lowercase_							)
				else:
								print(json.dumps(lowercase_						,   indent=2							)							)


if __name__ == "__main__":
			__SCREAMING_SNAKE_CASE  :						Optional[Any]    							=      parse_args()
			if OPTS.out_image_dir:
						import matplotlib

						matplotlib.use('''Agg''')
						import matplotlib.pyplot as plt
			main()





"""simple docstring"""
import warnings

from ..trainer import Trainer
from ..utils import logging


__SCREAMING_SNAKE_CASE  :						Dict    							=      logging.get_logger(__name__)


class 							lowerCamelCase_(   A__    ):





					'''simple docstring'''



					def __init__(    self				,       lowerCamelCase__=None				,       **lowerCamelCase__   ):
									warnings.warn(
									    '''`SageMakerTrainer` is deprecated and will be removed in v5 of Transformers. You can use `Trainer` '''
									    '''instead.'''				,       lowerCamelCase__				,       )
									super().__init__(args=lowerCamelCase__				,       **lowerCamelCase__   )
