"""simple docstring"""




import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
__lowerCAmelCase    :							Tuple													=						{"LayoutLMv2Config", "LayoutLMv3Config"}

@is_pipeline_test
class       a_      (     unittest.TestCase			):
					UpperCamelCase_ :	Optional[int]			=    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
					UpperCamelCase_ :	Optional[int]			=    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

					if model_mapping is not None:
										UpperCamelCase_ :	Any			=    {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
					if tf_model_mapping is not None:
										UpperCamelCase_ :	int			=    {
										    config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
										}




					@require_torch
					def 						_SCREAMING_SNAKE_CASE						(	self						:     str     ):
							lowerCAmelCase__        =   pipeline(
							    task="""text-classification"""  ,						model="""hf-internal-testing/tiny-random-distilbert"""  ,						framework="""pt"""     )

							lowerCAmelCase__        =   text_classifier("""This is great !"""     )
							self.assertEqual(nested_simplify(snake_case__     )  ,						[{"""label""": """LABEL_0""", """score""": 0.504}]     )

							lowerCAmelCase__        =   text_classifier("""This is great !"""  ,						top_k=2     )
							self.assertEqual(
							    nested_simplify(snake_case__     )  ,						[{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}]     )

							lowerCAmelCase__        =   text_classifier(["""This is great !""", """This is bad"""]  ,						top_k=2     )
							self.assertEqual(
							    nested_simplify(snake_case__     )  ,						[
							        [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}],
							        [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}],
							    ]  ,						)

							lowerCAmelCase__        =   text_classifier("""This is great !"""  ,						top_k=1     )
							self.assertEqual(nested_simplify(snake_case__     )  ,						[{"""label""": """LABEL_0""", """score""": 0.504}]     )

							# Legacy behavior
							lowerCAmelCase__        =   text_classifier("""This is great !"""  ,						return_all_scores=snake_case__     )
							self.assertEqual(nested_simplify(snake_case__     )  ,						[{"""label""": """LABEL_0""", """score""": 0.504}]     )

							lowerCAmelCase__        =   text_classifier("""This is great !"""  ,						return_all_scores=snake_case__     )
							self.assertEqual(
							    nested_simplify(snake_case__     )  ,						[[{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}]]     )

							lowerCAmelCase__        =   text_classifier(["""This is great !""", """Something else"""]  ,						return_all_scores=snake_case__     )
							self.assertEqual(
							    nested_simplify(snake_case__     )  ,						[
							        [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}],
							        [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}],
							    ]  ,						)

							lowerCAmelCase__        =   text_classifier(["""This is great !""", """Something else"""]  ,						return_all_scores=snake_case__     )
							self.assertEqual(
							    nested_simplify(snake_case__     )  ,						[
							        {"""label""": """LABEL_0""", """score""": 0.504},
							        {"""label""": """LABEL_0""", """score""": 0.504},
							    ]  ,						)




					@require_torch
					def 						_SCREAMING_SNAKE_CASE						(	self						:     str     ):
							import torch

							lowerCAmelCase__        =   pipeline(
							    task="""text-classification"""  ,						model="""hf-internal-testing/tiny-random-distilbert"""  ,						framework="""pt"""  ,						device=torch.device("""cpu"""     )  ,						)

							lowerCAmelCase__        =   text_classifier("""This is great !"""     )
							self.assertEqual(nested_simplify(snake_case__     )  ,						[{"""label""": """LABEL_0""", """score""": 0.504}]     )




					@require_tf
					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]     ):
							lowerCAmelCase__        =   pipeline(
							    task="""text-classification"""  ,						model="""hf-internal-testing/tiny-random-distilbert"""  ,						framework="""tf"""     )

							lowerCAmelCase__        =   text_classifier("""This is great !"""     )
							self.assertEqual(nested_simplify(snake_case__     )  ,						[{"""label""": """LABEL_0""", """score""": 0.504}]     )




					@slow
					@require_torch
					def 						_SCREAMING_SNAKE_CASE						(	self						:     Optional[int]     ):
							lowerCAmelCase__        =   pipeline("""text-classification"""     )

							lowerCAmelCase__        =   text_classifier("""This is great !"""     )
							self.assertEqual(nested_simplify(snake_case__     )  ,						[{"""label""": """POSITIVE""", """score""": 1.0}]     )
							lowerCAmelCase__        =   text_classifier("""This is bad !"""     )
							self.assertEqual(nested_simplify(snake_case__     )  ,						[{"""label""": """NEGATIVE""", """score""": 1.0}]     )
							lowerCAmelCase__        =   text_classifier("""Birds are a type of animal"""     )
							self.assertEqual(nested_simplify(snake_case__     )  ,						[{"""label""": """POSITIVE""", """score""": 0.988}]     )




					@slow
					@require_tf
					def 						_SCREAMING_SNAKE_CASE						(	self						:     str     ):
							lowerCAmelCase__        =   pipeline("""text-classification"""  ,						framework="""tf"""     )

							lowerCAmelCase__        =   text_classifier("""This is great !"""     )
							self.assertEqual(nested_simplify(snake_case__     )  ,						[{"""label""": """POSITIVE""", """score""": 1.0}]     )
							lowerCAmelCase__        =   text_classifier("""This is bad !"""     )
							self.assertEqual(nested_simplify(snake_case__     )  ,						[{"""label""": """NEGATIVE""", """score""": 1.0}]     )
							lowerCAmelCase__        =   text_classifier("""Birds are a type of animal"""     )
							self.assertEqual(nested_simplify(snake_case__     )  ,						[{"""label""": """POSITIVE""", """score""": 0.988}]     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]  ,						snake_case__						:     Optional[int]  ,						snake_case__						:     Any  ,						snake_case__						:     List[str]     ):
							lowerCAmelCase__        =   TextClassificationPipeline(model=snake_case__  ,						tokenizer=snake_case__     )
							return text_classifier, ["HuggingFace is in", "This is another test"]




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Optional[int]  ,						snake_case__						:     Tuple  ,						snake_case__						:     Dict     ):
							lowerCAmelCase__        =   text_classifier.model
							# Small inputs because BartTokenizer tiny has maximum position embeddings = 22
							lowerCAmelCase__        =   """HuggingFace is in"""
							lowerCAmelCase__        =   text_classifier(snake_case__     )

							self.assertEqual(nested_simplify(snake_case__     )  ,						[{"""label""": ANY(snake_case__     ), """score""": ANY(snake_case__     )}]     )
							self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()     )

							lowerCAmelCase__        =   ["""HuggingFace is in """, """Paris is in France"""]
							lowerCAmelCase__        =   text_classifier(snake_case__     )
							self.assertEqual(
							    nested_simplify(snake_case__     )  ,						[{"""label""": ANY(snake_case__     ), """score""": ANY(snake_case__     )}, {"""label""": ANY(snake_case__     ), """score""": ANY(snake_case__     )}]  ,						)
							self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()     )
							self.assertTrue(outputs[1]["""label"""] in model.config.idalabel.values()     )

							# Forcing to get all results with `top_k=None`
							# This is NOT the legacy format
							lowerCAmelCase__        =   text_classifier(snake_case__  ,						top_k=snake_case__     )
							lowerCAmelCase__        =   len(model.config.idalabel.values()     )
							self.assertEqual(
							    nested_simplify(snake_case__     )  ,						[[{"""label""": ANY(snake_case__     ), """score""": ANY(snake_case__     )}] * N, [{"""label""": ANY(snake_case__     ), """score""": ANY(snake_case__     )}] * N]  ,						)

							lowerCAmelCase__        =   {"""text""": """HuggingFace is in """, """text_pair""": """Paris is in France"""}
							lowerCAmelCase__        =   text_classifier(snake_case__     )
							self.assertEqual(
							    nested_simplify(snake_case__     )  ,						{"""label""": ANY(snake_case__     ), """score""": ANY(snake_case__     )}  ,						)
							self.assertTrue(outputs["""label"""] in model.config.idalabel.values()     )

							# This might be used a text pair, but tokenizer + pipe interaction
							# makes it hard to understand that it's not using the pair properly
							# https://github.com/huggingface/transformers/issues/17305
							# We disabled this usage instead as it was outputting wrong outputs.
							lowerCAmelCase__        =   [["""HuggingFace is in """, """Paris is in France"""]]
							with self.assertRaises(snake_case__     ):
									text_classifier(snake_case__     )

							# This used to be valid for doing text pairs
							# We're keeping it working because of backward compatibility
							lowerCAmelCase__        =   text_classifier([[["""HuggingFace is in """, """Paris is in France"""]]]     )
							self.assertEqual(
							    nested_simplify(snake_case__     )  ,						[{"""label""": ANY(snake_case__     ), """score""": ANY(snake_case__     )}]  ,						)
							self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()     )





"""simple docstring"""




import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
			from sagemaker import Session, TrainingJobAnalytics
			from sagemaker.huggingface import HuggingFace

@pytest.mark.skipif(
    literal_eval(os.getenv("TEST_SAGEMAKER" ,		"False"			)			) is not True ,		reason="Skipping test because should only be run when releasing minor transformers version" ,		)
@pytest.mark.usefixtures("sm_env"			)
@parameterized_class(
    [
        {
            "framework": "pytorch",
            "script": "run_glue_model_parallelism.py",
            "model_name_or_path": "roberta-large",
            "instance_type": "ml.p3dn.24xlarge",
            "results": {"train_runtime": 1600, "eval_accuracy": 0.3, "eval_loss": 1.2},
        },
        {
            "framework": "pytorch",
            "script": "run_glue.py",
            "model_name_or_path": "roberta-large",
            "instance_type": "ml.p3dn.24xlarge",
            "results": {"train_runtime": 1600, "eval_accuracy": 0.3, "eval_loss": 1.2},
        },
    ]			)
class       a_      (     unittest.TestCase			):




					def 						_SCREAMING_SNAKE_CASE						(	self						:     int     ):
							if self.framework == "pytorch":
									subprocess.run(
									    F"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split()  ,						encoding="""utf-8"""  ,						check=snake_case__  ,						)
							assert hasattr(self  ,						"""env"""     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     str  ,						snake_case__						:     Optional[Any]     ):
							# configuration for running training on smdistributed Model Parallel
							lowerCAmelCase__        =   {
							    """enabled""": True,
							    """processes_per_host""": 8,
							}
							lowerCAmelCase__        =   {
							    """enabled""": True,
							    """parameters""": {
							        """microbatches""": 4,
							        """placement_strategy""": """spread""",
							        """pipeline""": """interleaved""",
							        """optimize""": """speed""",
							        """partitions""": 4,
							        """ddp""": True,
							    },
							}

							lowerCAmelCase__        =   {"""smdistributed""": {"""modelparallel""": smp_options}, """mpi""": mpi_options}

							lowerCAmelCase__        =   """trainer""" if self.script == """run_glue.py""" else """smtrainer"""
							# creates estimator
							return HuggingFace(
							    entry_point=self.script  ,						source_dir=self.env.test_path  ,						role=self.env.role  ,						image_uri=self.env.image_uri  ,						base_job_name=F"""{self.env.base_job_name}-{instance_count}-smp-{name_extension}"""  ,						instance_count=snake_case__  ,						instance_type=self.instance_type  ,						debugger_hook_config=snake_case__  ,						hyperparameters={
							        **self.env.hyperparameters,
							        """model_name_or_path""": self.model_name_or_path,
							        """max_steps""": 500,
							    }  ,						metric_definitions=self.env.metric_definitions  ,						distribution=snake_case__  ,						py_version="""py36"""  ,						)




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]  ,						snake_case__						:     str     ):
							TrainingJobAnalytics(snake_case__     ).export_csv(F"""{self.env.test_path}/{job_name}_metrics.csv"""     )




					@parameterized.expand([(1,)]     )
					def 						_SCREAMING_SNAKE_CASE						(	self						:     List[Any]  ,						snake_case__						:     List[str]     ):
							# create estimator
							lowerCAmelCase__        =   self.create_estimator(snake_case__     )

							# run training
							estimator.fit()

							# result dataframe
							lowerCAmelCase__        =   TrainingJobAnalytics(estimator.latest_training_job.name     ).dataframe()

							# extract kpis
							lowerCAmelCase__        =   list(result_metrics_df[result_metrics_df.metric_name == """eval_accuracy"""]["""value"""]     )
							lowerCAmelCase__        =   list(result_metrics_df[result_metrics_df.metric_name == """eval_loss"""]["""value"""]     )
							# get train time from SageMaker job, this includes starting, preprocessing, stopping
							lowerCAmelCase__        =   (
							    Session().describe_training_job(estimator.latest_training_job.name     ).get("""TrainingTimeInSeconds"""  ,						999999     )
							)

							# assert kpis
							assert train_runtime <= self.results["train_runtime"]
							assert all(t >= self.results["""eval_accuracy"""] for t in eval_accuracy     )
							assert all(t <= self.results["""eval_loss"""] for t in eval_loss     )

							# dump tests result into json file to share in PR
							with open(F"""{estimator.latest_training_job.name}.json"""  ,						"""w"""     ) as outfile:
									json.dump({"""train_time""": train_runtime, """eval_accuracy""": eval_accuracy, """eval_loss""": eval_loss}  ,						snake_case__     )




