





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_lowerCamelCase			       =      re.compile(r'\b(a|an|the)\b', re.UNICODE)

_lowerCamelCase			       =      None




def    __UpperCAmelCase(      ):
							_lowerCamelCase							:     Union[str, Any]	=		argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''      )
							parser.add_argument('''data_file''' ,    metavar='''data.json''' ,    help='''Input data JSON file.'''      )
							parser.add_argument('''pred_file''' ,    metavar='''pred.json''' ,    help='''Model predictions.'''      )
							parser.add_argument(
							    '''--out-file''' ,    '''-o''' ,    metavar='''eval.json''' ,    help='''Write accuracy metrics to file (default is stdout).'''      )
							parser.add_argument(
							    '''--na-prob-file''' ,    '''-n''' ,    metavar='''na_prob.json''' ,    help='''Model estimates of probability of no answer.'''      )
							parser.add_argument(
							    '''--na-prob-thresh''' ,    '''-t''' ,    type=lowercase_ ,    default=1.0 ,    help='''Predict "" if no-answer probability exceeds this (default = 1.0).''' ,    )
							parser.add_argument(
							    '''--out-image-dir''' ,    '''-p''' ,    metavar='''out_images''' ,    default=lowercase_ ,    help='''Save precision-recall curves to directory.'''      )
							parser.add_argument('''--verbose''' ,    '''-v''' ,    action='''store_true'''      )
							if len(sys.argv      ) == 1:
														parser.print_help()
														sys.exit(1      )
							return parser.parse_args()




def    __UpperCAmelCase(      lowercase_      ):
							_lowerCamelCase							:     Union[str, Any]	=		{}
							for article in dataset:
														for p in article["paragraphs"]:
																					for qa in p["qas"]:
																												_lowerCamelCase							:     Union[str, Any]	=		bool(qa['''answers''']['''text''']      )
							return qid_to_has_ans




def    __UpperCAmelCase(      lowercase_      ):

							def remove_articles(lowercase_      ):
														return ARTICLES_REGEX.sub(''' ''' ,    lowercase_      )

							def white_space_fix(lowercase_      ):
														return " ".join(text.split()      )

							def remove_punc(lowercase_      ):
														_lowerCamelCase							:     List[str]	=		set(string.punctuation      )
														return "".join(ch for ch in text if ch not in exclude      )

							def lower(lowercase_      ):
														return text.lower()

							return white_space_fix(remove_articles(remove_punc(lower(lowercase_      )      )      )      )




def    __UpperCAmelCase(      lowercase_      ):
							if not s:
														return []
							return normalize_answer(lowercase_      ).split()




def    __UpperCAmelCase(      lowercase_ ,    lowercase_      ):
							return int(normalize_answer(lowercase_      ) == normalize_answer(lowercase_      )      )




def    __UpperCAmelCase(      lowercase_ ,    lowercase_      ):
							_lowerCamelCase							:     int	=		get_tokens(lowercase_      )
							_lowerCamelCase							:     List[Any]	=		get_tokens(lowercase_      )
							_lowerCamelCase							:     Optional[int]	=		collections.Counter(lowercase_      ) & collections.Counter(lowercase_      )
							_lowerCamelCase							:     str	=		sum(common.values()      )
							if len(lowercase_      ) == 0 or len(lowercase_      ) == 0:
														# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
														return int(gold_toks == pred_toks      )
							if num_same == 0:
														return 0
							_lowerCamelCase							:     int	=		1.0 * num_same / len(lowercase_      )
							_lowerCamelCase							:     Optional[Any]	=		1.0 * num_same / len(lowercase_      )
							_lowerCamelCase							:     Tuple	=		(2 * precision * recall) / (precision + recall)
							return fa




def    __UpperCAmelCase(      lowercase_ ,    lowercase_      ):
							_lowerCamelCase							:     int	=		{}
							_lowerCamelCase							:     str	=		{}
							for article in dataset:
														for p in article["paragraphs"]:
																					for qa in p["qas"]:
																												_lowerCamelCase							:     Dict	=		qa['''id''']
																												_lowerCamelCase							:     Tuple	=		[t for t in qa['''answers''']['''text'''] if normalize_answer(lowercase_      )]
																												if not gold_answers:
																																			# For unanswerable questions, only correct answer is empty string
																																			_lowerCamelCase							:     Tuple	=		['''''']
																												if qid not in preds:
																																			print(F"""Missing prediction for {qid}"""      )
																																			continue
																												_lowerCamelCase							:     List[str]	=		preds[qid]
																												# Take max over all gold answers
																												_lowerCamelCase							:     Any	=		max(compute_exact(lowercase_ ,    lowercase_      ) for a in gold_answers      )
																												_lowerCamelCase							:     Optional[int]	=		max(compute_fa(lowercase_ ,    lowercase_      ) for a in gold_answers      )
							return exact_scores, fa_scores




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      ):
							_lowerCamelCase							:     List[str]	=		{}
							for qid, s in scores.items():
														_lowerCamelCase							:     int	=		na_probs[qid] > na_prob_thresh
														if pred_na:
																					_lowerCamelCase							:     Optional[int]	=		float(not qid_to_has_ans[qid]      )
														else:
																					_lowerCamelCase							:     List[str]	=		s
							return new_scores




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_=None      ):
							if not qid_list:
														_lowerCamelCase							:     Optional[int]	=		len(lowercase_      )
														return collections.OrderedDict(
														    [
														        ('''exact''', 1_00.0 * sum(exact_scores.values()      ) / total),
														        ('''f1''', 1_00.0 * sum(fa_scores.values()      ) / total),
														        ('''total''', total),
														    ]      )
							else:
														_lowerCamelCase							:     int	=		len(lowercase_      )
														return collections.OrderedDict(
														    [
														        ('''exact''', 1_00.0 * sum(exact_scores[k] for k in qid_list      ) / total),
														        ('''f1''', 1_00.0 * sum(fa_scores[k] for k in qid_list      ) / total),
														        ('''total''', total),
														    ]      )




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_      ):
							for k in new_eval:
														_lowerCamelCase							:     Union[str, Any]	=		new_eval[k]




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      ):
							plt.step(lowercase_ ,    lowercase_ ,    color='''b''' ,    alpha=0.2 ,    where='''post'''      )
							plt.fill_between(lowercase_ ,    lowercase_ ,    step='''post''' ,    alpha=0.2 ,    color='''b'''      )
							plt.xlabel('''Recall'''      )
							plt.ylabel('''Precision'''      )
							plt.xlim([0.0, 1.0_5]      )
							plt.ylim([0.0, 1.0_5]      )
							plt.title(lowercase_      )
							plt.savefig(lowercase_      )
							plt.clf()




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_=None ,    lowercase_=None      ):
							_lowerCamelCase							:     Optional[int]	=		sorted(lowercase_ ,    key=lambda lowercase_      : na_probs[k]      )
							_lowerCamelCase							:     Any	=		0.0
							_lowerCamelCase							:     Dict	=		1.0
							_lowerCamelCase							:     str	=		0.0
							_lowerCamelCase							:     Tuple	=		[1.0]
							_lowerCamelCase							:     Optional[Any]	=		[0.0]
							_lowerCamelCase							:     str	=		0.0
							for i, qid in enumerate(lowercase_      ):
														if qid_to_has_ans[qid]:
																					true_pos += scores[qid]
														_lowerCamelCase							:     Dict	=		true_pos / float(i + 1      )
														_lowerCamelCase							:     List[Any]	=		true_pos / float(lowercase_      )
														if i == len(lowercase_      ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
																					# i.e., if we can put a threshold after this point
																					avg_prec += cur_p * (cur_r - recalls[-1])
																					precisions.append(lowercase_      )
																					recalls.append(lowercase_      )
							if out_image:
														plot_pr_curve(lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      )
							return {"ap": 1_00.0 * avg_prec}




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      ):
							if out_image_dir and not os.path.exists(lowercase_      ):
														os.makedirs(lowercase_      )
							_lowerCamelCase							:     List[Any]	=		sum(1 for v in qid_to_has_ans.values() if v      )
							if num_true_pos == 0:
														return
							_lowerCamelCase							:     List[str]	=		make_precision_recall_eval(
							    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    out_image=os.path.join(lowercase_ ,    '''pr_exact.png'''      ) ,    title='''Precision-Recall curve for Exact Match score''' ,    )
							_lowerCamelCase							:     Optional[int]	=		make_precision_recall_eval(
							    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    out_image=os.path.join(lowercase_ ,    '''pr_f1.png'''      ) ,    title='''Precision-Recall curve for F1 score''' ,    )
							_lowerCamelCase							:     List[str]	=		{k: float(lowercase_      ) for k, v in qid_to_has_ans.items()}
							_lowerCamelCase							:     List[str]	=		make_precision_recall_eval(
							    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    out_image=os.path.join(lowercase_ ,    '''pr_oracle.png'''      ) ,    title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)''' ,    )
							merge_eval(lowercase_ ,    lowercase_ ,    '''pr_exact'''      )
							merge_eval(lowercase_ ,    lowercase_ ,    '''pr_f1'''      )
							merge_eval(lowercase_ ,    lowercase_ ,    '''pr_oracle'''      )




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      ):
							if not qid_list:
														return
							_lowerCamelCase							:     int	=		[na_probs[k] for k in qid_list]
							_lowerCamelCase							:     Optional[int]	=		np.ones_like(lowercase_      ) / float(len(lowercase_      )      )
							plt.hist(lowercase_ ,    weights=lowercase_ ,    bins=20 ,    range=(0.0, 1.0)      )
							plt.xlabel('''Model probability of no-answer'''      )
							plt.ylabel('''Proportion of dataset'''      )
							plt.title(F"""Histogram of no-answer probability: {name}"""      )
							plt.savefig(os.path.join(lowercase_ ,    F"""na_prob_hist_{name}.png"""      )      )
							plt.clf()




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      ):
							_lowerCamelCase							:     Optional[int]	=		sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]      )
							_lowerCamelCase							:     str	=		num_no_ans
							_lowerCamelCase							:     Tuple	=		cur_score
							_lowerCamelCase							:     Union[str, Any]	=		0.0
							_lowerCamelCase							:     Tuple	=		sorted(lowercase_ ,    key=lambda lowercase_      : na_probs[k]      )
							for i, qid in enumerate(lowercase_      ):
														if qid not in scores:
																					continue
														if qid_to_has_ans[qid]:
																					_lowerCamelCase							:     Any	=		scores[qid]
														else:
																					if preds[qid]:
																												_lowerCamelCase							:     Any	=		-1
																					else:
																												_lowerCamelCase							:     Tuple	=		0
														cur_score += diff
														if cur_score > best_score:
																					_lowerCamelCase							:     Union[str, Any]	=		cur_score
																					_lowerCamelCase							:     Tuple	=		na_probs[qid]
							return 1_00.0 * best_score / len(lowercase_      ), best_thresh




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      ):
							_lowerCamelCase							:     Optional[int]	=		find_best_thresh(lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      )
							_lowerCamelCase							:     List[Any]	=		find_best_thresh(lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      )
							_lowerCamelCase							:     List[Any]	=		best_exact
							_lowerCamelCase							:     int	=		exact_thresh
							_lowerCamelCase							:     Optional[Any]	=		best_fa
							_lowerCamelCase							:     List[str]	=		fa_thresh




def    __UpperCAmelCase(      ):
							with open(OPTS.data_file      ) as f:
														_lowerCamelCase							:     Any	=		json.load(lowercase_      )
														_lowerCamelCase							:     Dict	=		dataset_json['''data''']
							with open(OPTS.pred_file      ) as f:
														_lowerCamelCase							:     Tuple	=		json.load(lowercase_      )
							if OPTS.na_prob_file:
														with open(OPTS.na_prob_file      ) as f:
																					_lowerCamelCase							:     Tuple	=		json.load(lowercase_      )
							else:
														_lowerCamelCase							:     str	=		{k: 0.0 for k in preds}
							_lowerCamelCase							:     List[Any]	=		make_qid_to_has_ans(lowercase_      )  # maps qid to True/False
							_lowerCamelCase							:     Optional[int]	=		[k for k, v in qid_to_has_ans.items() if v]
							_lowerCamelCase							:     Union[str, Any]	=		[k for k, v in qid_to_has_ans.items() if not v]
							_lowerCamelCase							:     str	=		get_raw_scores(lowercase_ ,    lowercase_      )
							_lowerCamelCase							:     Optional[int]	=		apply_no_ans_threshold(lowercase_ ,    lowercase_ ,    lowercase_ ,    OPTS.na_prob_thresh      )
							_lowerCamelCase							:     Tuple	=		apply_no_ans_threshold(lowercase_ ,    lowercase_ ,    lowercase_ ,    OPTS.na_prob_thresh      )
							_lowerCamelCase							:     Dict	=		make_eval_dict(lowercase_ ,    lowercase_      )
							if has_ans_qids:
														_lowerCamelCase							:     Dict	=		make_eval_dict(lowercase_ ,    lowercase_ ,    qid_list=lowercase_      )
														merge_eval(lowercase_ ,    lowercase_ ,    '''HasAns'''      )
							if no_ans_qids:
														_lowerCamelCase							:     Tuple	=		make_eval_dict(lowercase_ ,    lowercase_ ,    qid_list=lowercase_      )
														merge_eval(lowercase_ ,    lowercase_ ,    '''NoAns'''      )
							if OPTS.na_prob_file:
														find_all_best_thresh(lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_      )
							if OPTS.na_prob_file and OPTS.out_image_dir:
														run_precision_recall_analysis(lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    lowercase_ ,    OPTS.out_image_dir      )
														histogram_na_prob(lowercase_ ,    lowercase_ ,    OPTS.out_image_dir ,    '''hasAns'''      )
														histogram_na_prob(lowercase_ ,    lowercase_ ,    OPTS.out_image_dir ,    '''noAns'''      )
							if OPTS.out_file:
														with open(OPTS.out_file ,    '''w'''      ) as f:
																					json.dump(lowercase_ ,    lowercase_      )
							else:
														print(json.dumps(lowercase_ ,    indent=2      )      )


if __name__ == "__main__":
							_lowerCamelCase			       =      parse_args()
							if OPTS.out_image_dir:
														import matplotlib

														matplotlib.use('Agg')
														import matplotlib.pyplot as plt
							main()








import importlib.util
import os
import platform
from argparse import ArgumentParser

import huggingface_hub

from .. import __version__ as version
from ..utils import (
    is_accelerate_available,
    is_flax_available,
    is_safetensors_available,
    is_tf_available,
    is_torch_available,
)
from . import BaseTransformersCLICommand




def    __UpperCAmelCase(      lowercase_      ):
       return EnvironmentCommand()




def    __UpperCAmelCase(      lowercase_      ):
       return EnvironmentCommand(args.accelerate_config_file      )







class 			__A       (      lowerCamelCase__  ):




      """simple docstring"""







      @staticmethod
      def 				__snake_case			(		a__):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		parser.add_parser('''env''')
             download_parser.set_defaults(func=a__)
             download_parser.add_argument(
                 '''--accelerate-config_file'''      ,							default=a__      ,							help='''The accelerate config file to use for the default values in the launching script.'''      ,							)
             download_parser.set_defaults(func=a__)


      def __init__(		self      ,							a__      ,							*a__):



             """simple docstring"""





             _lowerCamelCase							:     str	=		accelerate_config_file


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		'''not installed'''
             if is_safetensors_available():
                    import safetensors

                    _lowerCamelCase							:     Optional[Any]	=		safetensors.__version__
             elif importlib.util.find_spec('''safetensors''') is not None:
                    import safetensors

                    _lowerCamelCase							:     Optional[int]	=		F"""{safetensors.__version__} but is ignored because of PyTorch version too old."""

             _lowerCamelCase							:     Union[str, Any]	=		'''not installed'''
             _lowerCamelCase							:     Any	=		'''not found'''
             if is_accelerate_available():
                    import accelerate
                    from accelerate.commands.config import default_config_file, load_config_from_file

                    _lowerCamelCase							:     Optional[int]	=		accelerate.__version__
                    # Get the default from the config file.
                    if self._accelerate_config_file is not None or os.path.isfile(a__):
                           _lowerCamelCase							:     Optional[int]	=		load_config_from_file(self._accelerate_config_file).to_dict()

                    _lowerCamelCase							:     str	=		(
                        '''\n'''.join([F"""\t- {prop}: {val}""" for prop, val in accelerate_config.items()])
                        if isinstance(a__      ,							a__)
                        else F"""\t{accelerate_config}"""
                    )

             _lowerCamelCase							:     List[Any]	=		'''not installed'''
             _lowerCamelCase							:     Tuple	=		'''NA'''
             if is_torch_available():
                    import torch

                    _lowerCamelCase							:     int	=		torch.__version__
                    _lowerCamelCase							:     List[str]	=		torch.cuda.is_available()

             _lowerCamelCase							:     str	=		'''not installed'''
             _lowerCamelCase							:     Union[str, Any]	=		'''NA'''
             if is_tf_available():
                    import tensorflow as tf

                    _lowerCamelCase							:     List[str]	=		tf.__version__
                    try:
                           # deprecated in v2.1
                           _lowerCamelCase							:     Optional[int]	=		tf.test.is_gpu_available()
                    except AttributeError:
                           # returns list of devices, convert to bool
                           _lowerCamelCase							:     Optional[int]	=		bool(tf.config.list_physical_devices('''GPU'''))

             _lowerCamelCase							:     str	=		'''not installed'''
             _lowerCamelCase							:     List[Any]	=		'''not installed'''
             _lowerCamelCase							:     List[Any]	=		'''not installed'''
             _lowerCamelCase							:     Optional[int]	=		'''NA'''
             if is_flax_available():
                    import flax
                    import jax
                    import jaxlib

                    _lowerCamelCase							:     Any	=		flax.__version__
                    _lowerCamelCase							:     str	=		jax.__version__
                    _lowerCamelCase							:     Any	=		jaxlib.__version__
                    _lowerCamelCase							:     int	=		jax.lib.xla_bridge.get_backend().platform

             _lowerCamelCase							:     int	=		{
                 '''`transformers` version''': version,
                 '''Platform''': platform.platform(),
                 '''Python version''': platform.python_version(),
                 '''Huggingface_hub version''': huggingface_hub.__version__,
                 '''Safetensors version''': F"""{safetensors_version}""",
                 '''Accelerate version''': F"""{accelerate_version}""",
                 '''Accelerate config''': F"""{accelerate_config_str}""",
                 '''PyTorch version (GPU?)''': F"""{pt_version} ({pt_cuda_available})""",
                 '''Tensorflow version (GPU?)''': F"""{tf_version} ({tf_cuda_available})""",
                 '''Flax version (CPU?/GPU?/TPU?)''': F"""{flax_version} ({jax_backend})""",
                 '''Jax version''': F"""{jax_version}""",
                 '''JaxLib version''': F"""{jaxlib_version}""",
                 '''Using GPU in script?''': '''<fill in>''',
                 '''Using distributed or parallel set-up in script?''': '''<fill in>''',
             }

             print('''\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n''')
             print(self.format_dict(a__))

             return info



      @staticmethod
      def 				__snake_case			(		a__):



             """simple docstring"""





             return "\n".join([F"""- {prop}: {val}""" for prop, val in d.items()]) + "\n"

