



'''simple docstring'''




import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
							from sagemaker import Session, TrainingJobAnalytics
							from sagemaker.huggingface import HuggingFace

@pytest.mark.skipif(
    literal_eval(os.getenv('TEST_SAGEMAKER'   ,  'False'    )    ) is not True   ,  reason='Skipping test because should only be run when releasing minor transformers version'   ,  )
@pytest.mark.usefixtures('sm_env'    )
@parameterized_class(
    [
        {
            'framework': 'pytorch',
            'script': 'run_glue.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 650, 'eval_accuracy': 0.7, 'eval_loss': 0.6},
        },
        {
            'framework': 'pytorch',
            'script': 'run_ddp.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 600, 'eval_accuracy': 0.7, 'eval_loss': 0.6},
        },
        {
            'framework': 'tensorflow',
            'script': 'run_tf_dist.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 600, 'eval_accuracy': 0.6, 'eval_loss': 0.7},
        },
    ]    )
class     UpperCamelCase_				(							unittest.TestCase    ):





	def 				_lowercase(						self					)    ->    int:
								if self.framework == "pytorch":
															subprocess.run(
															    f'''cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py'''.split()	,	encoding="""utf-8"""	,	check=A	,	)
								assert hasattr(self	,	"""env"""					)





	def 				_lowercase(						self	,	A					)    ->    str:
								UpperCAmelCase  :	Any							  =	f'''{self.env.base_job_name}-{instance_count}-{'ddp' if 'ddp' in self.script else 'smd'}'''
								# distributed data settings
								UpperCAmelCase  :	Tuple							  =	{"""smdistributed""": {"""dataparallel""": {"""enabled""": True}}} if self.script != """run_ddp.py""" else None

								# creates estimator
								return HuggingFace(
								    entry_point=self.script	,	source_dir=self.env.test_path	,	role=self.env.role	,	image_uri=self.env.image_uri	,	base_job_name=A	,	instance_count=A	,	instance_type=self.instance_type	,	debugger_hook_config=A	,	hyperparameters={**self.env.distributed_hyperparameters, """model_name_or_path""": self.model_name_or_path}	,	metric_definitions=self.env.metric_definitions	,	distribution=A	,	py_version="""py36"""	,	)





	def 				_lowercase(						self	,	A					)    ->    Any:
								TrainingJobAnalytics(A					).export_csv(f'''{self.env.test_path}/{job_name}_metrics.csv'''					)






	@parameterized.expand([(2,)]					)
	def 				_lowercase(						self	,	A					)    ->    str:
								# create estimator
								UpperCAmelCase  :	Union[str, Any]							  =	self.create_estimator(A					)

								# run training
								estimator.fit()

								# result dataframe
								UpperCAmelCase  :	str							  =	TrainingJobAnalytics(estimator.latest_training_job.name					).dataframe()

								# extract kpis
								UpperCAmelCase  :	Optional[Any]							  =	list(result_metrics_df[result_metrics_df.metric_name == """eval_accuracy"""]["""value"""]					)
								UpperCAmelCase  :	Any							  =	list(result_metrics_df[result_metrics_df.metric_name == """eval_loss"""]["""value"""]					)
								# get train time from SageMaker job, this includes starting, preprocessing, stopping
								UpperCAmelCase  :	List[Any]							  =	(
								    Session().describe_training_job(estimator.latest_training_job.name					).get("""TrainingTimeInSeconds"""	,	999999					)
								)

								# assert kpis
								assert train_runtime <= self.results["train_runtime"]
								assert all(t >= self.results["""eval_accuracy"""] for t in eval_accuracy					)
								assert all(t <= self.results["""eval_loss"""] for t in eval_loss					)

								# dump tests result into json file to share in PR
								with open(f'''{estimator.latest_training_job.name}.json'''	,	"""w"""					) as outfile:
															json.dump({"""train_time""": train_runtime, """eval_accuracy""": eval_accuracy, """eval_loss""": eval_loss}	,	A					)






'''simple docstring'''




from typing import List, Union

import numpy as np

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, logging
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, ChunkPipeline


a     :    List[str]      						=					logging.get_logger(__name__)

class     UpperCamelCase_				(							__magic_name__    ):





	def 				_lowercase(						self	,	A					)    ->    Optional[int]:
								if isinstance(A	,	A					):
															UpperCAmelCase  :	Union[str, Any]							  =	[label.strip() for label in labels.split(""","""					) if label.strip()]
								return labels






	def __call__(						self	,	A	,	A	,	A					)    ->    str:
								if len(A					) == 0 or len(A					) == 0:
															raise ValueError("""You must include at least one label and at least one sequence."""					)
								if hypothesis_template.format(labels[0]					) == hypothesis_template:
															raise ValueError(
															    (
															        """The provided hypothesis_template \"{}\" was not able to be formatted with the target labels. """
															        """Make sure the passed template includes formatting syntax such as {{}} where the label should go."""
															    ).format(A					)					)

								if isinstance(A	,	A					):
															UpperCAmelCase  :	Tuple							  =	[sequences]

								UpperCAmelCase  :	Optional[Any]							  =	[]
								for sequence in sequences:
															sequence_pairs.extend([[sequence, hypothesis_template.format(A					)] for label in labels]					)

								return sequence_pairs, sequences




@add_end_docstrings(__magic_name__    )
class     UpperCamelCase_				(							__magic_name__    ):





	def __init__(						self	,	A=ZeroShotClassificationArgumentHandler()	,	*A	,	**A					)    ->    Optional[int]:
								UpperCAmelCase  :	Tuple							  =	args_parser
								super().__init__(*A	,	**A					)
								if self.entailment_id == -1:
															logger.warning(
															    """Failed to determine 'entailment' label id from the label2id mapping in the model config. Setting to """
															    """-1. Define a descriptive label2id mapping in the model config to ensure correct outputs."""					)





	@property
	def 				_lowercase(						self					)    ->    List[Any]:
								for label, ind in self.model.config.labelaid.items():
															if label.lower().startswith("""entail"""					):
																						return ind
								return -1





	def 				_lowercase(						self	,	A	,	A=True	,	A=True	,	A=TruncationStrategy.ONLY_FIRST	,	**A					)    ->    str:
								UpperCAmelCase  :	Tuple							  =	self.framework
								if self.tokenizer.pad_token is None:
															# Override for tokenizers not supporting padding
															logger.error(
															    """Tokenizer was not supporting padding necessary for zero-shot, attempting to use """
															    """ `pad_token=eos_token`"""					)
															UpperCAmelCase  :	Any							  =	self.tokenizer.eos_token
								try:
															UpperCAmelCase  :	Tuple							  =	self.tokenizer(
															    A	,	add_special_tokens=A	,	return_tensors=A	,	padding=A	,	truncation=A	,	)
								except Exception as e:
															if "too short" in str(A					):
																						# tokenizers might yell that we want to truncate
																						# to a value that is not even reached by the input.
																						# In that case we don't want to truncate.
																						# It seems there's not a really better way to catch that
																						# exception.

																						UpperCAmelCase  :	List[str]							  =	self.tokenizer(
																						    A	,	add_special_tokens=A	,	return_tensors=A	,	padding=A	,	truncation=TruncationStrategy.DO_NOT_TRUNCATE	,	)
															else:
																						raise e

								return inputs





	def 				_lowercase(						self	,	**A					)    ->    Tuple:
								if kwargs.get("""multi_class"""	,	A					) is not None:
															UpperCAmelCase  :	Any							  =	kwargs["""multi_class"""]
															logger.warning(
															    """The `multi_class` argument has been deprecated and renamed to `multi_label`. """
															    """`multi_class` will be removed in a future version of Transformers."""					)
								UpperCAmelCase  :	int							  =	{}
								if "candidate_labels" in kwargs:
															UpperCAmelCase  :	Tuple							  =	self._args_parser._parse_labels(kwargs["""candidate_labels"""]					)
								if "hypothesis_template" in kwargs:
															UpperCAmelCase  :	List[Any]							  =	kwargs["""hypothesis_template"""]

								UpperCAmelCase  :	Dict							  =	{}
								if "multi_label" in kwargs:
															UpperCAmelCase  :	Union[str, Any]							  =	kwargs["""multi_label"""]
								return preprocess_params, {}, postprocess_params





	def __call__(						self	,	A	,	*A	,	**A	,	)    ->    Tuple:
								if len(A					) == 0:
															pass
								elif len(A					) == 1 and "candidate_labels" not in kwargs:
															UpperCAmelCase  :	Optional[Any]							  =	args[0]
								else:
															raise ValueError(f'''Unable to understand extra arguments {args}'''					)

								return super().__call__(A	,	**A					)





	def 				_lowercase(						self	,	A	,	A=None	,	A="This example is {}."					)    ->    List[Any]:
								UpperCAmelCase  ,  UpperCAmelCase  :	List[Any]							  =	self._args_parser(A	,	A	,	A					)

								for i, (candidate_label, sequence_pair) in enumerate(zip(A	,	A					)					):
															UpperCAmelCase  :	Any							  =	self._parse_and_tokenize([sequence_pair]					)

															yield {
															    "candidate_label": candidate_label,
															    "sequence": sequences[0],
															    "is_last": i == len(A					) - 1,
															    **model_input,
															}





	def 				_lowercase(						self	,	A					)    ->    Optional[int]:
								UpperCAmelCase  :	Optional[Any]							  =	inputs["""candidate_label"""]
								UpperCAmelCase  :	Tuple							  =	inputs["""sequence"""]
								UpperCAmelCase  :	List[Any]							  =	{k: inputs[k] for k in self.tokenizer.model_input_names}
								UpperCAmelCase  :	Tuple							  =	self.model(**A					)

								UpperCAmelCase  :	Optional[int]							  =	{
								    """candidate_label""": candidate_label,
								    """sequence""": sequence,
								    """is_last""": inputs["""is_last"""],
								    **outputs,
								}
								return model_outputs






	def 				_lowercase(						self	,	A	,	A=False					)    ->    List[str]:
								UpperCAmelCase  :	Dict							  =	[outputs["""candidate_label"""] for outputs in model_outputs]
								UpperCAmelCase  :	List[Any]							  =	[outputs["""sequence"""] for outputs in model_outputs]
								UpperCAmelCase  :	List[Any]							  =	np.concatenate([output["""logits"""].numpy() for output in model_outputs]					)
								UpperCAmelCase  :	Optional[Any]							  =	logits.shape[0]
								UpperCAmelCase  :	int							  =	len(A					)
								UpperCAmelCase  :	List[Any]							  =	N // n
								UpperCAmelCase  :	int							  =	logits.reshape((num_sequences, n, -1)					)

								if multi_label or len(A					) == 1:
															# softmax over the entailment vs. contradiction dim for each label independently
															UpperCAmelCase  :	str							  =	self.entailment_id
															UpperCAmelCase  :	str							  =	-1 if entailment_id == 0 else 0
															UpperCAmelCase  :	Optional[Any]							  =	reshaped_outputs[..., [contradiction_id, entailment_id]]
															UpperCAmelCase  :	int							  =	np.exp(A					) / np.exp(A					).sum(-1	,	keepdims=A					)
															UpperCAmelCase  :	int							  =	scores[..., 1]
								else:
															# softmax the "entailment" logits over all candidate labels
															UpperCAmelCase  :	Dict							  =	reshaped_outputs[..., self.entailment_id]
															UpperCAmelCase  :	Optional[int]							  =	np.exp(A					) / np.exp(A					).sum(-1	,	keepdims=A					)

								UpperCAmelCase  :	int							  =	list(reversed(scores[0].argsort()					)					)
								return {
								    "sequence": sequences[0],
								    "labels": [candidate_labels[i] for i in top_inds],
								    "scores": scores[0, top_inds].tolist(),
								}

