


'''simple docstring'''







import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCAmelCase    :     Optional[Any]					=						re.compile(r"""\b(a|an|the)\b""", re.UNICODE)

lowerCAmelCase    :     int					=						None






def 			lowercase				():







       """simple docstring"""







       _lowerCAmelCase    :							Tuple   			=       argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'					)
       parser.add_argument('data_file'   ,						metavar='data.json'   ,						help='Input data JSON file.'					)
       parser.add_argument('pred_file'   ,						metavar='pred.json'   ,						help='Model predictions.'					)
       parser.add_argument(
           '--out-file'   ,						'-o'   ,						metavar='eval.json'   ,						help='Write accuracy metrics to file (default is stdout).'					)
       parser.add_argument(
           '--na-prob-file'   ,						'-n'   ,						metavar='na_prob.json'   ,						help='Model estimates of probability of no answer.'					)
       parser.add_argument(
           '--na-prob-thresh'   ,						'-t'   ,						type=_A   ,						default=1.0   ,						help='Predict "" if no-answer probability exceeds this (default = 1.0).'   ,						)
       parser.add_argument(
           '--out-image-dir'   ,						'-p'   ,						metavar='out_images'   ,						default=_A   ,						help='Save precision-recall curves to directory.'					)
       parser.add_argument('--verbose'   ,						'-v'   ,						action='store_true'					)
       if len(sys.argv					) == 1:
              parser.print_help()
              sys.exit(1					)
       return parser.parse_args()






def 			lowercase				(_A					):







       """simple docstring"""







       _lowerCAmelCase    :							Union[str, Any]   			=       {}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            _lowerCAmelCase    :							Dict   			=       bool(qa['answers']['text']					)
       return qid_to_has_ans






def 			lowercase				(_A					):







       """simple docstring"""







       def remove_articles(_A					):
              return ARTICLES_REGEX.sub(' '   ,						_A					)

       def white_space_fix(_A					):
              return " ".join(text.split()					)

       def remove_punc(_A					):
              _lowerCAmelCase    :							Optional[int]   			=       set(string.punctuation					)
              return "".join(ch for ch in text if ch not in exclude					)

       def lower(_A					):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(_A					)					)					)					)






def 			lowercase				(_A					):







       """simple docstring"""







       if not s:
              return []
       return normalize_answer(_A					).split()






def 			lowercase				(_A   ,						_A					):







       """simple docstring"""







       return int(normalize_answer(_A					) == normalize_answer(_A					)					)






def 			lowercase				(_A   ,						_A					):







       """simple docstring"""







       _lowerCAmelCase    :							Tuple   			=       get_tokens(_A					)
       _lowerCAmelCase    :							Optional[Any]   			=       get_tokens(_A					)
       _lowerCAmelCase    :							Tuple   			=       collections.Counter(_A					) & collections.Counter(_A					)
       _lowerCAmelCase    :							Any   			=       sum(common.values()					)
       if len(_A					) == 0 or len(_A					) == 0:
              # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
              return int(gold_toks == pred_toks					)
       if num_same == 0:
              return 0
       _lowerCAmelCase    :							Tuple   			=       1.0 * num_same / len(_A					)
       _lowerCAmelCase    :							Dict   			=       1.0 * num_same / len(_A					)
       _lowerCAmelCase    :							Optional[Any]   			=       (2 * precision * recall) / (precision + recall)
       return fa






def 			lowercase				(_A   ,						_A					):







       """simple docstring"""







       _lowerCAmelCase    :							Optional[Any]   			=       {}
       _lowerCAmelCase    :							int   			=       {}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            _lowerCAmelCase    :							str   			=       qa['id']
                            _lowerCAmelCase    :							Optional[Any]   			=       [t for t in qa['answers']['text'] if normalize_answer(_A					)]
                            if not gold_answers:
                                   # For unanswerable questions, only correct answer is empty string
                                   _lowerCAmelCase    :							Tuple   			=       ['']
                            if qid not in preds:
                                   print(f'Missing prediction for {qid}'					)
                                   continue
                            _lowerCAmelCase    :							Optional[int]   			=       preds[qid]
                            # Take max over all gold answers
                            _lowerCAmelCase    :							List[str]   			=       max(compute_exact(_A   ,						_A					) for a in gold_answers					)
                            _lowerCAmelCase    :							int   			=       max(compute_fa(_A   ,						_A					) for a in gold_answers					)
       return exact_scores, fa_scores






def 			lowercase				(_A   ,						_A   ,						_A   ,						_A					):







       """simple docstring"""







       _lowerCAmelCase    :							Tuple   			=       {}
       for qid, s in scores.items():
              _lowerCAmelCase    :							str   			=       na_probs[qid] > na_prob_thresh
              if pred_na:
                     _lowerCAmelCase    :							List[str]   			=       float(not qid_to_has_ans[qid]					)
              else:
                     _lowerCAmelCase    :							Dict   			=       s
       return new_scores






def 			lowercase				(_A   ,						_A   ,						_A=None					):







       """simple docstring"""







       if not qid_list:
              _lowerCAmelCase    :							Tuple   			=       len(_A					)
              return collections.OrderedDict(
                  [
                      ('exact', 100.0 * sum(exact_scores.values()					) / total),
                      ('f1', 100.0 * sum(fa_scores.values()					) / total),
                      ('total', total),
                  ]					)
       else:
              _lowerCAmelCase    :							List[Any]   			=       len(_A					)
              return collections.OrderedDict(
                  [
                      ('exact', 100.0 * sum(exact_scores[k] for k in qid_list					) / total),
                      ('f1', 100.0 * sum(fa_scores[k] for k in qid_list					) / total),
                      ('total', total),
                  ]					)






def 			lowercase				(_A   ,						_A   ,						_A					):







       """simple docstring"""







       for k in new_eval:
              _lowerCAmelCase    :							Optional[int]   			=       new_eval[k]






def 			lowercase				(_A   ,						_A   ,						_A   ,						_A					):







       """simple docstring"""







       plt.step(_A   ,						_A   ,						color='b'   ,						alpha=0.2   ,						where='post'					)
       plt.fill_between(_A   ,						_A   ,						step='post'   ,						alpha=0.2   ,						color='b'					)
       plt.xlabel('Recall'					)
       plt.ylabel('Precision'					)
       plt.xlim([0.0, 1.05]					)
       plt.ylim([0.0, 1.05]					)
       plt.title(_A					)
       plt.savefig(_A					)
       plt.clf()






def 			lowercase				(_A   ,						_A   ,						_A   ,						_A   ,						_A=None   ,						_A=None					):







       """simple docstring"""







       _lowerCAmelCase    :							str   			=       sorted(_A   ,						key=lambda _A					: na_probs[k]					)
       _lowerCAmelCase    :							List[str]   			=       0.0
       _lowerCAmelCase    :							Union[str, Any]   			=       1.0
       _lowerCAmelCase    :							List[str]   			=       0.0
       _lowerCAmelCase    :							Union[str, Any]   			=       [1.0]
       _lowerCAmelCase    :							Any   			=       [0.0]
       _lowerCAmelCase    :							List[str]   			=       0.0
       for i, qid in enumerate(_A					):
              if qid_to_has_ans[qid]:
                     true_pos += scores[qid]
              _lowerCAmelCase    :							int   			=       true_pos / float(i + 1					)
              _lowerCAmelCase    :							List[str]   			=       true_pos / float(_A					)
              if i == len(_A					) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                     # i.e., if we can put a threshold after this point
                     avg_prec += cur_p * (cur_r - recalls[-1])
                     precisions.append(_A					)
                     recalls.append(_A					)
       if out_image:
              plot_pr_curve(_A   ,						_A   ,						_A   ,						_A					)
       return {"ap": 100.0 * avg_prec}






def 			lowercase				(_A   ,						_A   ,						_A   ,						_A   ,						_A   ,						_A					):







       """simple docstring"""







       if out_image_dir and not os.path.exists(_A					):
              os.makedirs(_A					)
       _lowerCAmelCase    :							int   			=       sum(1 for v in qid_to_has_ans.values() if v					)
       if num_true_pos == 0:
              return
       _lowerCAmelCase    :							Optional[int]   			=       make_precision_recall_eval(
           _A   ,						_A   ,						_A   ,						_A   ,						out_image=os.path.join(_A   ,						'pr_exact.png'					)   ,						title='Precision-Recall curve for Exact Match score'   ,						)
       _lowerCAmelCase    :							Any   			=       make_precision_recall_eval(
           _A   ,						_A   ,						_A   ,						_A   ,						out_image=os.path.join(_A   ,						'pr_f1.png'					)   ,						title='Precision-Recall curve for F1 score'   ,						)
       _lowerCAmelCase    :							List[Any]   			=       {k: float(_A					) for k, v in qid_to_has_ans.items()}
       _lowerCAmelCase    :							Any   			=       make_precision_recall_eval(
           _A   ,						_A   ,						_A   ,						_A   ,						out_image=os.path.join(_A   ,						'pr_oracle.png'					)   ,						title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'   ,						)
       merge_eval(_A   ,						_A   ,						'pr_exact'					)
       merge_eval(_A   ,						_A   ,						'pr_f1'					)
       merge_eval(_A   ,						_A   ,						'pr_oracle'					)






def 			lowercase				(_A   ,						_A   ,						_A   ,						_A					):







       """simple docstring"""







       if not qid_list:
              return
       _lowerCAmelCase    :							List[Any]   			=       [na_probs[k] for k in qid_list]
       _lowerCAmelCase    :							Dict   			=       np.ones_like(_A					) / float(len(_A					)					)
       plt.hist(_A   ,						weights=_A   ,						bins=2_0   ,						range=(0.0, 1.0)					)
       plt.xlabel('Model probability of no-answer'					)
       plt.ylabel('Proportion of dataset'					)
       plt.title(f'Histogram of no-answer probability: {name}'					)
       plt.savefig(os.path.join(_A   ,						f'na_prob_hist_{name}.png'					)					)
       plt.clf()






def 			lowercase				(_A   ,						_A   ,						_A   ,						_A					):







       """simple docstring"""







       _lowerCAmelCase    :							str   			=       sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]					)
       _lowerCAmelCase    :							Tuple   			=       num_no_ans
       _lowerCAmelCase    :							Union[str, Any]   			=       cur_score
       _lowerCAmelCase    :							str   			=       0.0
       _lowerCAmelCase    :							Tuple   			=       sorted(_A   ,						key=lambda _A					: na_probs[k]					)
       for i, qid in enumerate(_A					):
              if qid not in scores:
                     continue
              if qid_to_has_ans[qid]:
                     _lowerCAmelCase    :							Tuple   			=       scores[qid]
              else:
                     if preds[qid]:
                            _lowerCAmelCase    :							Dict   			=       -1
                     else:
                            _lowerCAmelCase    :							Tuple   			=       0
              cur_score += diff
              if cur_score > best_score:
                     _lowerCAmelCase    :							Any   			=       cur_score
                     _lowerCAmelCase    :							Union[str, Any]   			=       na_probs[qid]
       return 100.0 * best_score / len(_A					), best_thresh






def 			lowercase				(_A   ,						_A   ,						_A   ,						_A   ,						_A   ,						_A					):







       """simple docstring"""







       _lowerCAmelCase			,     _lowerCAmelCase    :							Tuple   			=       find_best_thresh(_A   ,						_A   ,						_A   ,						_A					)
       _lowerCAmelCase			,     _lowerCAmelCase    :							int   			=       find_best_thresh(_A   ,						_A   ,						_A   ,						_A					)
       _lowerCAmelCase    :							Optional[Any]   			=       best_exact
       _lowerCAmelCase    :							Any   			=       exact_thresh
       _lowerCAmelCase    :							Any   			=       best_fa
       _lowerCAmelCase    :							Any   			=       fa_thresh






def 			lowercase				():







       """simple docstring"""







       with open(OPTS.data_file					) as f:
              _lowerCAmelCase    :							Tuple   			=       json.load(_A					)
              _lowerCAmelCase    :							List[str]   			=       dataset_json['data']
       with open(OPTS.pred_file					) as f:
              _lowerCAmelCase    :							str   			=       json.load(_A					)
       if OPTS.na_prob_file:
              with open(OPTS.na_prob_file					) as f:
                     _lowerCAmelCase    :							Tuple   			=       json.load(_A					)
       else:
              _lowerCAmelCase    :							Any   			=       {k: 0.0 for k in preds}
       _lowerCAmelCase    :							Optional[int]   			=       make_qid_to_has_ans(_A					)  # maps qid to True/False
       _lowerCAmelCase    :							List[Any]   			=       [k for k, v in qid_to_has_ans.items() if v]
       _lowerCAmelCase    :							Union[str, Any]   			=       [k for k, v in qid_to_has_ans.items() if not v]
       _lowerCAmelCase			,     _lowerCAmelCase    :							Tuple   			=       get_raw_scores(_A   ,						_A					)
       _lowerCAmelCase    :							List[Any]   			=       apply_no_ans_threshold(_A   ,						_A   ,						_A   ,						OPTS.na_prob_thresh					)
       _lowerCAmelCase    :							Union[str, Any]   			=       apply_no_ans_threshold(_A   ,						_A   ,						_A   ,						OPTS.na_prob_thresh					)
       _lowerCAmelCase    :							Any   			=       make_eval_dict(_A   ,						_A					)
       if has_ans_qids:
              _lowerCAmelCase    :							Optional[int]   			=       make_eval_dict(_A   ,						_A   ,						qid_list=_A					)
              merge_eval(_A   ,						_A   ,						'HasAns'					)
       if no_ans_qids:
              _lowerCAmelCase    :							Optional[Any]   			=       make_eval_dict(_A   ,						_A   ,						qid_list=_A					)
              merge_eval(_A   ,						_A   ,						'NoAns'					)
       if OPTS.na_prob_file:
              find_all_best_thresh(_A   ,						_A   ,						_A   ,						_A   ,						_A   ,						_A					)
       if OPTS.na_prob_file and OPTS.out_image_dir:
              run_precision_recall_analysis(_A   ,						_A   ,						_A   ,						_A   ,						_A   ,						OPTS.out_image_dir					)
              histogram_na_prob(_A   ,						_A   ,						OPTS.out_image_dir   ,						'hasAns'					)
              histogram_na_prob(_A   ,						_A   ,						OPTS.out_image_dir   ,						'noAns'					)
       if OPTS.out_file:
              with open(OPTS.out_file   ,						'w'					) as f:
                     json.dump(_A   ,						_A					)
       else:
              print(json.dumps(_A   ,						indent=2					)					)


if __name__ == "__main__":
       lowerCAmelCase    :     Tuple					=						parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use("""Agg""")
              import matplotlib.pyplot as plt
       main()





'''simple docstring'''







from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowerCAmelCase    :     Optional[Any]					=						logging.get_logger(__name__)

lowerCAmelCase    :     str					=						{
    """EleutherAI/gpt-j-6B""": """https://huggingface.co/EleutherAI/gpt-j-6B/resolve/main/config.json""",
    # See all GPT-J models at https://huggingface.co/models?filter=gpt_j
}
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     __magic_name__    			=    "gptj"
     __magic_name__    			=    {
         "max_position_embeddings": "n_positions",
         "hidden_size": "n_embd",
         "num_attention_heads": "n_head",
         "num_hidden_layers": "n_layer",
     }







     def __init__( self      ,	snake_case__=5_0400      ,	snake_case__=2048      ,	snake_case__=4096      ,	snake_case__=28      ,	snake_case__=16      ,	snake_case__=64      ,	snake_case__=None      ,	snake_case__="gelu_new"      ,	snake_case__=0.0      ,	snake_case__=0.0      ,	snake_case__=0.0      ,	snake_case__=1E-5      ,	snake_case__=0.02      ,	snake_case__=True      ,	snake_case__=5_0256      ,	snake_case__=5_0256      ,	snake_case__=False      ,	**snake_case__      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							int   			=       vocab_size
            _lowerCAmelCase    :							Optional[int]   			=       n_positions
            _lowerCAmelCase    :							Optional[int]   			=       n_embd
            _lowerCAmelCase    :							Optional[int]   			=       n_layer
            _lowerCAmelCase    :							str   			=       n_head
            _lowerCAmelCase    :							Tuple   			=       n_inner
            _lowerCAmelCase    :							Tuple   			=       rotary_dim
            _lowerCAmelCase    :							Optional[int]   			=       activation_function
            _lowerCAmelCase    :							Any   			=       resid_pdrop
            _lowerCAmelCase    :							List[str]   			=       embd_pdrop
            _lowerCAmelCase    :							int   			=       attn_pdrop
            _lowerCAmelCase    :							Any   			=       layer_norm_epsilon
            _lowerCAmelCase    :							Optional[int]   			=       initializer_range
            _lowerCAmelCase    :							List[str]   			=       use_cache

            _lowerCAmelCase    :							Dict   			=       bos_token_id
            _lowerCAmelCase    :							Any   			=       eos_token_id

            super().__init__(
                bos_token_id=snake_case__      ,	eos_token_id=snake_case__      ,	tie_word_embeddings=snake_case__      ,	**snake_case__   )
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     def __init__( self      ,	snake_case__      ,	snake_case__ = "default"      ,	snake_case__ = None      ,	snake_case__ = False      ,	):

            '''simple docstring'''

            super().__init__(snake_case__      ,	task=snake_case__      ,	patching_specs=snake_case__      ,	use_past=snake_case__   )
            if not getattr(self._config      ,	'pad_token_id'      ,	snake_case__   ):
                   # TODO: how to do that better?
                   _lowerCAmelCase    :							Any   			=       0







     @property
     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							str   			=       OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}   )
            if self.use_past:
                   self.fill_with_past_key_values_(snake_case__      ,	direction='inputs'   )
                   _lowerCAmelCase    :							int   			=       {0: 'batch', 1: 'past_sequence + sequence'}
            else:
                   _lowerCAmelCase    :							int   			=       {0: 'batch', 1: 'sequence'}

            return common_inputs







     @property
     def a	( self   ):

            '''simple docstring'''

            return self._config.n_layer







     @property
     def a	( self   ):

            '''simple docstring'''

            return self._config.n_head







     def a	( self      ,	snake_case__      ,	snake_case__ = -1      ,	snake_case__ = -1      ,	snake_case__ = False      ,	snake_case__ = None      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[Any]   			=       super(snake_case__      ,	self   ).generate_dummy_inputs(
                snake_case__      ,	batch_size=snake_case__      ,	seq_length=snake_case__      ,	is_pair=snake_case__      ,	framework=snake_case__   )

            # We need to order the input in the way they appears in the forward()
            _lowerCAmelCase    :							Any   			=       OrderedDict({'input_ids': common_inputs['input_ids']}   )

            # Need to add the past_keys
            if self.use_past:
                   if not is_torch_available():
                          raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'   )
                   else:
                          import torch

                          _lowerCAmelCase			,     _lowerCAmelCase    :							Optional[Any]   			=       common_inputs['input_ids'].shape
                          # Not using the same length for past_key_values
                          _lowerCAmelCase    :							Any   			=       seqlen + 2
                          _lowerCAmelCase    :							Optional[int]   			=       (
                              batch,
                              self.num_attention_heads,
                              past_key_values_length,
                              self._config.hidden_size // self.num_attention_heads,
                          )
                          _lowerCAmelCase    :							Tuple   			=       [
                              (torch.zeros(snake_case__   ), torch.zeros(snake_case__   )) for _ in range(self.num_layers   )
                          ]

            _lowerCAmelCase    :							Tuple   			=       common_inputs['attention_mask']
            if self.use_past:
                   _lowerCAmelCase    :							Any   			=       ordered_inputs['attention_mask'].dtype
                   _lowerCAmelCase    :							Union[str, Any]   			=       torch.cat(
                       [ordered_inputs['attention_mask'], torch.ones(snake_case__      ,	snake_case__      ,	dtype=snake_case__   )]      ,	dim=1   )

            return ordered_inputs







     @property
     def a	( self   ):

            '''simple docstring'''

            return 13

