





'''simple docstring'''







import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
					from sagemaker import Session, TrainingJobAnalytics
					from sagemaker.huggingface import HuggingFace


@pytest.mark.skipif(
    literal_eval(os.getenv("TEST_SAGEMAKER"       ,		"False"     )     ) is not True       ,		reason="Skipping test because should only be run when releasing minor transformers version"       ,		)
@pytest.mark.usefixtures("sm_env"     )
@parameterized_class(
    [
        {
            "framework": "pytorch",
            "script": "run_glue.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.g4dn.xlarge",
            "results": {"train_runtime": 650, "eval_accuracy": 0.6, "eval_loss": 0.9},
        },
        {
            "framework": "tensorflow",
            "script": "run_tf.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.g4dn.xlarge",
            "results": {"train_runtime": 600, "eval_accuracy": 0.3, "eval_loss": 0.9},
        },
    ]     )
class 					__a      (unittest.TestCase     ):

							def     UpperCAmelCase__	(							self							:     Tuple   )					->       Optional[Any]:







													"""simple docstring"""
													if self.framework == "pytorch":
																			subprocess.run(
																			    F"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split()       ,     encoding='''utf-8'''       ,     check=__magic_name__       ,     )
													assert hasattr(self       ,     '''env'''   )

							def     UpperCAmelCase__	(							self							:     Dict       ,     __magic_name__							:     List[str]=1   )					->       Optional[int]:







													"""simple docstring"""
													# creates estimator
													return HuggingFace(
													    entry_point=self.script       ,     source_dir=self.env.test_path       ,     role=self.env.role       ,     image_uri=self.env.image_uri       ,     base_job_name=F"""{self.env.base_job_name}-single"""       ,     instance_count=__magic_name__       ,     instance_type=self.instance_type       ,     debugger_hook_config=__magic_name__       ,     hyperparameters={**self.env.hyperparameters, '''model_name_or_path''': self.model_name_or_path}       ,     metric_definitions=self.env.metric_definitions       ,     py_version='''py36'''       ,     )

							def     UpperCAmelCase__	(							self							:     Dict       ,     __magic_name__							:     List[Any]   )					->       Optional[Any]:







													"""simple docstring"""
													TrainingJobAnalytics(__magic_name__   ).export_csv(F"""{self.env.test_path}/{job_name}_metrics.csv"""   )



							def     UpperCAmelCase__	(							self							:     Union[str, Any]   )					->       Any:







													"""simple docstring"""
													# create estimator
													UpperCAmelCase_			:      Dict     						=  self.create_estimator()

													# run training
													estimator.fit()

													# result dataframe
													UpperCAmelCase_			:      Dict     						=  TrainingJobAnalytics(estimator.latest_training_job.name   ).dataframe()

													# extract kpis
													UpperCAmelCase_			:      Union[str, Any]     						=  list(result_metrics_df[result_metrics_df.metric_name == '''eval_accuracy''']['''value''']   )
													UpperCAmelCase_			:      int     						=  list(result_metrics_df[result_metrics_df.metric_name == '''eval_loss''']['''value''']   )
													# get train time from SageMaker job, this includes starting, preprocessing, stopping
													UpperCAmelCase_			:      int     						=  (
													    Session().describe_training_job(estimator.latest_training_job.name   ).get('''TrainingTimeInSeconds'''       ,     99_99_99   )
													)

													# assert kpis
													assert train_runtime <= self.results["train_runtime"]
													assert all(t >= self.results['''eval_accuracy'''] for t in eval_accuracy   )
													assert all(t <= self.results['''eval_loss'''] for t in eval_loss   )

													# dump tests result into json file to share in PR
													with open(F"""{estimator.latest_training_job.name}.json"""       ,     '''w'''   ) as outfile:
																			json.dump({'''train_time''': train_runtime, '''eval_accuracy''': eval_accuracy, '''eval_loss''': eval_loss}       ,     __magic_name__   )












'''simple docstring'''







import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
					import torch

					from transformers.generation import DisjunctiveConstraint


@require_torch
class 					__a      (unittest.TestCase     ):

							def     UpperCAmelCase__	(							self							:     Optional[Any]   )					->       List[Any]:







													"""simple docstring"""
													# For consistency across different places the DisjunctiveConstraint is called,
													# dc.token_ids is a list of integers. It is also initialized only by integers.

													UpperCAmelCase_			:      List[str]     						=  [[1, 2, 4], [1, 2, 3, 4]]
													UpperCAmelCase_			:      List[str]     						=  DisjunctiveConstraint(__magic_name__   )
													self.assertTrue(isinstance(dc.token_ids       ,     __magic_name__   )   )

													with self.assertRaises(__magic_name__   ):
																			DisjunctiveConstraint(torch.LongTensor([[1, 2, 4], [1, 2, 3]]   )   )

													with self.assertRaises(__magic_name__   ):
																			DisjunctiveConstraint([torch.LongTensor([1, 2, 4]   ), torch.LongTensor([1, 2, 3, 4, 5]   )]   )

							def     UpperCAmelCase__	(							self							:     List[str]   )					->       Dict:







													"""simple docstring"""
													# We can't have constraints that are complete subsets of another. This leads to a preverse
													# interpretation of "constraint fulfillment": does generating [1,2,3] fulfill the constraint?
													# It would mean that it generated [1,2] which fulfills it, but it's in the middle of potentially
													# fulfilling [1,2,3,4]. If we believe that [1,2,3] does fulfill the constraint, then the algorithm
													# will necessarily never reach [1,2,3,4], giving users a false sense of control (better to just not allow it).
													UpperCAmelCase_			:      Tuple     						=  [[1, 2], [1, 2, 3, 4]]

													with self.assertRaises(__magic_name__   ):
																			DisjunctiveConstraint(__magic_name__   )  # fails here

							def     UpperCAmelCase__	(							self							:     Optional[int]   )					->       Union[str, Any]:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[int]     						=  [[1, 2, 3], [1, 2, 4]]

													UpperCAmelCase_			:      List[str]     						=  DisjunctiveConstraint(__magic_name__   )

													UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      List[Any]     						=  dc.update(1   )
													UpperCAmelCase_			:      Dict     						=  stepped is True and completed is False and reset is False
													self.assertTrue(__magic_name__   )
													self.assertTrue(not dc.completed   )
													self.assertTrue(dc.current_seq == [1]   )

													UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      str     						=  dc.update(2   )
													UpperCAmelCase_			:      Optional[Any]     						=  stepped is True and completed is False and reset is False
													self.assertTrue(__magic_name__   )
													self.assertTrue(not dc.completed   )
													self.assertTrue(dc.current_seq == [1, 2]   )

													UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      List[Any]     						=  dc.update(3   )
													UpperCAmelCase_			:      Dict     						=  stepped is True and completed is True and reset is False
													self.assertTrue(__magic_name__   )
													self.assertTrue(dc.completed   )  # Completed!
													self.assertTrue(dc.current_seq == [1, 2, 3]   )



							def     UpperCAmelCase__	(							self							:     int   )					->       Dict:







													"""simple docstring"""
													UpperCAmelCase_			:      Any     						=  [[1, 2, 3], [1, 2, 4, 5], [1, 2, 5]]

													UpperCAmelCase_			:      Tuple     						=  DisjunctiveConstraint(__magic_name__   )

													UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      List[Any]     						=  dc.update(1   )
													self.assertTrue(not dc.completed   )
													self.assertTrue(dc.current_seq == [1]   )

													UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      Optional[int]     						=  dc.update(2   )
													self.assertTrue(not dc.completed   )
													self.assertTrue(dc.current_seq == [1, 2]   )

													UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      int     						=  dc.update(4   )
													self.assertTrue(not dc.completed   )
													self.assertTrue(dc.current_seq == [1, 2, 4]   )

													UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      int     						=  dc.update(5   )
													self.assertTrue(dc.completed   )  # Completed!
													self.assertTrue(dc.current_seq == [1, 2, 4, 5]   )

													dc.reset()

													UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      List[Any]     						=  dc.update(1   )
													self.assertTrue(not dc.completed   )
													self.assertTrue(dc.remaining() == 3   )
													self.assertTrue(dc.current_seq == [1]   )

													UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      Optional[Any]     						=  dc.update(2   )
													self.assertTrue(not dc.completed   )
													self.assertTrue(dc.remaining() == 2   )
													self.assertTrue(dc.current_seq == [1, 2]   )

													UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      Optional[int]     						=  dc.update(5   )
													self.assertTrue(dc.completed   )  # Completed!
													self.assertTrue(dc.remaining() == 0   )
													self.assertTrue(dc.current_seq == [1, 2, 5]   )





