


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCAmelCase							=						re.compile(R"""\b(a|an|the)\b""", re.UNICODE)

lowerCAmelCase							=						None




def 						__SCREAMING_SNAKE_CASE       (					)		->    Union[str, Any]:




						'''simple docstring'''







						__UpperCAmelCase :       Any  						=  argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''			)
						parser.add_argument('''data_file'''			,					metavar='''data.json'''			,					help='''Input data JSON file.'''			)
						parser.add_argument('''pred_file'''			,					metavar='''pred.json'''			,					help='''Model predictions.'''			)
						parser.add_argument(
						    '''--out-file'''			,					'''-o'''			,					metavar='''eval.json'''			,					help='''Write accuracy metrics to file (default is stdout).'''			)
						parser.add_argument(
						    '''--na-prob-file'''			,					'''-n'''			,					metavar='''na_prob.json'''			,					help='''Model estimates of probability of no answer.'''			)
						parser.add_argument(
						    '''--na-prob-thresh'''			,					'''-t'''			,					type=lowercase_			,					default=1.0			,					help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''			,					)
						parser.add_argument(
						    '''--out-image-dir'''			,					'''-p'''			,					metavar='''out_images'''			,					default=lowercase_			,					help='''Save precision-recall curves to directory.'''			)
						parser.add_argument('''--verbose'''			,					'''-v'''			,					action='''store_true'''			)
						if len(sys.argv			) == 1:
												parser.print_help()
												sys.exit(1			)
						return parser.parse_args()




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			)		->    List[str]:




						'''simple docstring'''







						__UpperCAmelCase :       Dict  						=  {}
						for article in dataset:
												for p in article["paragraphs"]:
																		for qa in p["qas"]:
																								__UpperCAmelCase :       Optional[Any]  						=  bool(qa['''answers''']['''text''']			)
						return qid_to_has_ans




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			)		->    List[str]:




						'''simple docstring'''







						def remove_articles(lowercase_			):
												return ARTICLES_REGEX.sub(''' '''			,					lowercase_			)

						def white_space_fix(lowercase_			):
												return " ".join(text.split()			)

						def remove_punc(lowercase_			):
												__UpperCAmelCase :       str  						=  set(string.punctuation			)
												return "".join(ch for ch in text if ch not in exclude			)

						def lower(lowercase_			):
												return text.lower()

						return white_space_fix(remove_articles(remove_punc(lower(lowercase_			)			)			)			)




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			)		->    Tuple:




						'''simple docstring'''







						if not s:
												return []
						return normalize_answer(lowercase_			).split()




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			)		->    List[str]:




						'''simple docstring'''







						return int(normalize_answer(lowercase_			) == normalize_answer(lowercase_			)			)




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			)		->    Optional[int]:




						'''simple docstring'''







						__UpperCAmelCase :       List[Any]  						=  get_tokens(lowercase_			)
						__UpperCAmelCase :       Optional[Any]  						=  get_tokens(lowercase_			)
						__UpperCAmelCase :       Union[str, Any]  						=  collections.Counter(lowercase_			) & collections.Counter(lowercase_			)
						__UpperCAmelCase :       List[Any]  						=  sum(common.values()			)
						if len(lowercase_			) == 0 or len(lowercase_			) == 0:
												# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
												return int(gold_toks == pred_toks			)
						if num_same == 0:
												return 0
						__UpperCAmelCase :       str  						=  1.0 * num_same / len(lowercase_			)
						__UpperCAmelCase :       Optional[int]  						=  1.0 * num_same / len(lowercase_			)
						__UpperCAmelCase :       List[str]  						=  (2 * precision * recall) / (precision + recall)
						return fa




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			)		->    int:




						'''simple docstring'''







						__UpperCAmelCase :       Dict  						=  {}
						__UpperCAmelCase :       List[str]  						=  {}
						for article in dataset:
												for p in article["paragraphs"]:
																		for qa in p["qas"]:
																								__UpperCAmelCase :       List[Any]  						=  qa['''id''']
																								__UpperCAmelCase :       Optional[int]  						=  [t for t in qa['''answers''']['''text'''] if normalize_answer(lowercase_			)]
																								if not gold_answers:
																														# For unanswerable questions, only correct answer is empty string
																														__UpperCAmelCase :       Union[str, Any]  						=  ['''''']
																								if qid not in preds:
																														print(f"Missing prediction for {qid}"			)
																														continue
																								__UpperCAmelCase :       str  						=  preds[qid]
																								# Take max over all gold answers
																								__UpperCAmelCase :       Tuple  						=  max(compute_exact(lowercase_			,					lowercase_			) for a in gold_answers			)
																								__UpperCAmelCase :       str  						=  max(compute_fa(lowercase_			,					lowercase_			) for a in gold_answers			)
						return exact_scores, fa_scores




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			)		->    Tuple:




						'''simple docstring'''







						__UpperCAmelCase :       Tuple  						=  {}
						for qid, s in scores.items():
												__UpperCAmelCase :       Tuple  						=  na_probs[qid] > na_prob_thresh
												if pred_na:
																		__UpperCAmelCase :       Union[str, Any]  						=  float(not qid_to_has_ans[qid]			)
												else:
																		__UpperCAmelCase :       List[str]  						=  s
						return new_scores




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			,					lowercase_=None			)		->    int:




						'''simple docstring'''







						if not qid_list:
												__UpperCAmelCase :       int  						=  len(lowercase_			)
												return collections.OrderedDict(
												    [
												        ('''exact''', 1_0_0.0 * sum(exact_scores.values()			) / total),
												        ('''f1''', 1_0_0.0 * sum(fa_scores.values()			) / total),
												        ('''total''', total),
												    ]			)
						else:
												__UpperCAmelCase :       int  						=  len(lowercase_			)
												return collections.OrderedDict(
												    [
												        ('''exact''', 1_0_0.0 * sum(exact_scores[k] for k in qid_list			) / total),
												        ('''f1''', 1_0_0.0 * sum(fa_scores[k] for k in qid_list			) / total),
												        ('''total''', total),
												    ]			)




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			,					lowercase_			)		->    Optional[Any]:




						'''simple docstring'''







						for k in new_eval:
												__UpperCAmelCase :       Optional[int]  						=  new_eval[k]




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			)		->    List[str]:




						'''simple docstring'''







						plt.step(lowercase_			,					lowercase_			,					color='''b'''			,					alpha=0.2			,					where='''post'''			)
						plt.fill_between(lowercase_			,					lowercase_			,					step='''post'''			,					alpha=0.2			,					color='''b'''			)
						plt.xlabel('''Recall'''			)
						plt.ylabel('''Precision'''			)
						plt.xlim([0.0, 1.0_5]			)
						plt.ylim([0.0, 1.0_5]			)
						plt.title(lowercase_			)
						plt.savefig(lowercase_			)
						plt.clf()




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_=None			,					lowercase_=None			)		->    Optional[Any]:




						'''simple docstring'''







						__UpperCAmelCase :       Dict  						=  sorted(lowercase_			,					key=lambda lowercase_			: na_probs[k]			)
						__UpperCAmelCase :       Union[str, Any]  						=  0.0
						__UpperCAmelCase :       int  						=  1.0
						__UpperCAmelCase :       str  						=  0.0
						__UpperCAmelCase :       str  						=  [1.0]
						__UpperCAmelCase :       str  						=  [0.0]
						__UpperCAmelCase :       Union[str, Any]  						=  0.0
						for i, qid in enumerate(lowercase_			):
												if qid_to_has_ans[qid]:
																		true_pos += scores[qid]
												__UpperCAmelCase :       Dict  						=  true_pos / float(i + 1			)
												__UpperCAmelCase :       List[Any]  						=  true_pos / float(lowercase_			)
												if i == len(lowercase_			) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
																		# i.e., if we can put a threshold after this point
																		avg_prec += cur_p * (cur_r - recalls[-1])
																		precisions.append(lowercase_			)
																		recalls.append(lowercase_			)
						if out_image:
												plot_pr_curve(lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			)
						return {"ap": 1_0_0.0 * avg_prec}




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			)		->    List[str]:




						'''simple docstring'''







						if out_image_dir and not os.path.exists(lowercase_			):
												os.makedirs(lowercase_			)
						__UpperCAmelCase :       str  						=  sum(1 for v in qid_to_has_ans.values() if v			)
						if num_true_pos == 0:
												return
						__UpperCAmelCase :       Tuple  						=  make_precision_recall_eval(
						    lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			,					out_image=os.path.join(lowercase_			,					'''pr_exact.png'''			)			,					title='''Precision-Recall curve for Exact Match score'''			,					)
						__UpperCAmelCase :       Dict  						=  make_precision_recall_eval(
						    lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			,					out_image=os.path.join(lowercase_			,					'''pr_f1.png'''			)			,					title='''Precision-Recall curve for F1 score'''			,					)
						__UpperCAmelCase :       List[str]  						=  {k: float(lowercase_			) for k, v in qid_to_has_ans.items()}
						__UpperCAmelCase :       List[str]  						=  make_precision_recall_eval(
						    lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			,					out_image=os.path.join(lowercase_			,					'''pr_oracle.png'''			)			,					title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''			,					)
						merge_eval(lowercase_			,					lowercase_			,					'''pr_exact'''			)
						merge_eval(lowercase_			,					lowercase_			,					'''pr_f1'''			)
						merge_eval(lowercase_			,					lowercase_			,					'''pr_oracle'''			)




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			)		->    Any:




						'''simple docstring'''







						if not qid_list:
												return
						__UpperCAmelCase :       Tuple  						=  [na_probs[k] for k in qid_list]
						__UpperCAmelCase :       str  						=  np.ones_like(lowercase_			) / float(len(lowercase_			)			)
						plt.hist(lowercase_			,					weights=lowercase_			,					bins=20			,					range=(0.0, 1.0)			)
						plt.xlabel('''Model probability of no-answer'''			)
						plt.ylabel('''Proportion of dataset'''			)
						plt.title(f"Histogram of no-answer probability: {name}"			)
						plt.savefig(os.path.join(lowercase_			,					f"na_prob_hist_{name}.png"			)			)
						plt.clf()




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			)		->    Any:




						'''simple docstring'''







						__UpperCAmelCase :       Any  						=  sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]			)
						__UpperCAmelCase :       Any  						=  num_no_ans
						__UpperCAmelCase :       Optional[Any]  						=  cur_score
						__UpperCAmelCase :       List[str]  						=  0.0
						__UpperCAmelCase :       Tuple  						=  sorted(lowercase_			,					key=lambda lowercase_			: na_probs[k]			)
						for i, qid in enumerate(lowercase_			):
												if qid not in scores:
																		continue
												if qid_to_has_ans[qid]:
																		__UpperCAmelCase :       Tuple  						=  scores[qid]
												else:
																		if preds[qid]:
																								__UpperCAmelCase :       Optional[Any]  						=  -1
																		else:
																								__UpperCAmelCase :       str  						=  0
												cur_score += diff
												if cur_score > best_score:
																		__UpperCAmelCase :       Optional[int]  						=  cur_score
																		__UpperCAmelCase :       int  						=  na_probs[qid]
						return 1_0_0.0 * best_score / len(lowercase_			), best_thresh




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			)		->    Dict:




						'''simple docstring'''







						__UpperCAmelCase						,			__UpperCAmelCase :       Union[str, Any]  						=  find_best_thresh(lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			)
						__UpperCAmelCase						,			__UpperCAmelCase :       Optional[int]  						=  find_best_thresh(lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			)
						__UpperCAmelCase :       int  						=  best_exact
						__UpperCAmelCase :       List[Any]  						=  exact_thresh
						__UpperCAmelCase :       Union[str, Any]  						=  best_fa
						__UpperCAmelCase :       Optional[Any]  						=  fa_thresh




def 						__SCREAMING_SNAKE_CASE       (					)		->    Dict:




						'''simple docstring'''







						with open(OPTS.data_file			) as f:
												__UpperCAmelCase :       List[Any]  						=  json.load(lowercase_			)
												__UpperCAmelCase :       List[Any]  						=  dataset_json['''data''']
						with open(OPTS.pred_file			) as f:
												__UpperCAmelCase :       str  						=  json.load(lowercase_			)
						if OPTS.na_prob_file:
												with open(OPTS.na_prob_file			) as f:
																		__UpperCAmelCase :       Optional[int]  						=  json.load(lowercase_			)
						else:
												__UpperCAmelCase :       List[str]  						=  {k: 0.0 for k in preds}
						__UpperCAmelCase :       int  						=  make_qid_to_has_ans(lowercase_			)  # maps qid to True/False
						__UpperCAmelCase :       int  						=  [k for k, v in qid_to_has_ans.items() if v]
						__UpperCAmelCase :       List[Any]  						=  [k for k, v in qid_to_has_ans.items() if not v]
						__UpperCAmelCase						,			__UpperCAmelCase :       Dict  						=  get_raw_scores(lowercase_			,					lowercase_			)
						__UpperCAmelCase :       List[str]  						=  apply_no_ans_threshold(lowercase_			,					lowercase_			,					lowercase_			,					OPTS.na_prob_thresh			)
						__UpperCAmelCase :       Optional[Any]  						=  apply_no_ans_threshold(lowercase_			,					lowercase_			,					lowercase_			,					OPTS.na_prob_thresh			)
						__UpperCAmelCase :       Any  						=  make_eval_dict(lowercase_			,					lowercase_			)
						if has_ans_qids:
												__UpperCAmelCase :       Tuple  						=  make_eval_dict(lowercase_			,					lowercase_			,					qid_list=lowercase_			)
												merge_eval(lowercase_			,					lowercase_			,					'''HasAns'''			)
						if no_ans_qids:
												__UpperCAmelCase :       List[str]  						=  make_eval_dict(lowercase_			,					lowercase_			,					qid_list=lowercase_			)
												merge_eval(lowercase_			,					lowercase_			,					'''NoAns'''			)
						if OPTS.na_prob_file:
												find_all_best_thresh(lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			)
						if OPTS.na_prob_file and OPTS.out_image_dir:
												run_precision_recall_analysis(lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			,					lowercase_			,					OPTS.out_image_dir			)
												histogram_na_prob(lowercase_			,					lowercase_			,					OPTS.out_image_dir			,					'''hasAns'''			)
												histogram_na_prob(lowercase_			,					lowercase_			,					OPTS.out_image_dir			,					'''noAns'''			)
						if OPTS.out_file:
												with open(OPTS.out_file			,					'''w'''			) as f:
																		json.dump(lowercase_			,					lowercase_			)
						else:
												print(json.dumps(lowercase_			,					indent=2			)			)


if __name__ == "__main__":
				lowerCAmelCase							=						parse_args()
				if OPTS.out_image_dir:
								import matplotlib

								matplotlib.use("""Agg""")
								import matplotlib.pyplot as plt
				main()





def 						__SCREAMING_SNAKE_CASE       (					)		->    list[list[int]]:




						'''simple docstring'''







						return [list(range(1000 - i			,					-1000 - i			,					-1			)			) for i in range(1000			)]


lowerCAmelCase							=						generate_large_matrix()
lowerCAmelCase							=						(
    [[4, 3, 2, -1], [3, 2, 1, -1], [1, 1, -1, -2], [-1, -1, -2, -3]],
    [[3, 2], [1, 0]],
    [[7, 7, 6]],
    [[7, 7, 6], [-1, -2, -3]],
    grid,
)




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			)		->    None:




						'''simple docstring'''







						assert all(row == sorted(lowercase_			,					reverse=lowercase_			) for row in grid			)
						assert all(list(lowercase_			) == sorted(lowercase_			,					reverse=lowercase_			) for col in zip(*lowercase_			)			)




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			)		->    int:




						'''simple docstring'''







						__UpperCAmelCase :       Dict  						=  0
						__UpperCAmelCase :       List[Any]  						=  len(lowercase_			) - 1

						# Edge cases such as no values or all numbers are negative.
						if not array or array[0] < 0:
												return 0

						while right + 1 > left:
												__UpperCAmelCase :       List[Any]  						=  (left + right) // 2
												__UpperCAmelCase :       Dict  						=  array[mid]

												# Num must be negative and the index must be greater than or equal to 0.
												if num < 0 and array[mid - 1] >= 0:
																		return mid

												if num >= 0:
																		__UpperCAmelCase :       Dict  						=  mid + 1
												else:
																		__UpperCAmelCase :       Optional[Any]  						=  mid - 1
    # No negative numbers so return the last index of the array + 1 which is the length.
						return len(lowercase_			)




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			)		->    int:




						'''simple docstring'''







						__UpperCAmelCase :       int  						=  0
						__UpperCAmelCase :       Dict  						=  len(grid[0]			)

						for i in range(len(lowercase_			)			):
												__UpperCAmelCase :       Any  						=  find_negative_index(grid[i][:bound]			)
												total += bound
						return (len(lowercase_			) * len(grid[0]			)) - total




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			)		->    int:




						'''simple docstring'''







						return len([number for row in grid for number in row if number < 0]			)




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			)		->    int:




						'''simple docstring'''







						__UpperCAmelCase :       List[Any]  						=  0
						for row in grid:
												for i, number in enumerate(lowercase_			):
																		if number < 0:
																								total += len(lowercase_			) - i
																								break
						return total




def 						__SCREAMING_SNAKE_CASE       (					)		->    None:




						'''simple docstring'''







						from timeit import timeit

						print('''Running benchmarks'''			)
						__UpperCAmelCase :       Tuple  						=  (
						    '''from __main__ import count_negatives_binary_search, '''
						    '''count_negatives_brute_force, count_negatives_brute_force_with_break, grid'''
						)
						for func in (
						    "count_negatives_binary_search",  # took 0.7727 seconds
						    "count_negatives_brute_force_with_break",  # took 4.6505 seconds
						    "count_negatives_brute_force",  # took 12.8160 seconds
						):
												__UpperCAmelCase :       Union[str, Any]  						=  timeit(f"{func}(grid=grid)"			,					setup=lowercase_			,					number=500			)
												print(f"{func}() took {time:0.4f} seconds"			)


if __name__ == "__main__":
				import doctest

				doctest.testmod()
				benchmark()

