'''simple docstring'''


import faiss  # noqa: F401 # Here to have a nice missing dependency error message early on
import numpy  # noqa: F401 # Here to have a nice missing dependency error message early on
import requests  # noqa: F401 # Here to have a nice missing dependency error message early on
import sklearn  # noqa: F401 # Here to have a nice missing dependency error message early on
import tqdm  # noqa: F401 # Here to have a nice missing dependency error message early on
from mauve import compute_mauve  # From: mauve-text

import datasets


__lowerCAmelCase	:       List[str]						       =	"\\n@inproceedings{pillutla-etal:mauve:neurips2021,\n  title={MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers},\n  author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid},\n  booktitle = {NeurIPS},\n  year      = {2021}\n}\n\n"

__lowerCAmelCase	:       Optional[Any]						       =	"\\nMAUVE is a library built on PyTorch and HuggingFace Transformers to measure the gap between neural text and human text with the eponymous MAUVE measure.\n\nMAUVE summarizes both Type I and Type II errors measured softly using Kullback–Leibler (KL) divergences.\n\nFor details, see the MAUVE paper: https://arxiv.org/abs/2102.01454 (Neurips, 2021).\n\nThis metrics is a wrapper around the official implementation of MAUVE:\nhttps://github.com/krishnap25/mauve\n"

__lowerCAmelCase	:       int						       =	"\nCalculates MAUVE scores between two lists of generated text and reference text.\nArgs:\n    predictions: list of generated text to score. Each predictions\n        should be a string with tokens separated by spaces.\n    references: list of reference for each prediction. Each\n        reference should be a string with tokens separated by spaces.\nOptional Args:\n    num_buckets: the size of the histogram to quantize P and Q. Options: 'auto' (default) or an integer\n    pca_max_data: the number data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. Default -1\n    kmeans_explained_var: amount of variance of the data to keep in dimensionality reduction by PCA. Default 0.9\n    kmeans_num_redo: number of times to redo k-means clustering (the best objective is kept). Default 5\n    kmeans_max_iter: maximum number of k-means iterations. Default 500\n    featurize_model_name: name of the model from which features are obtained. Default 'gpt2-large' Use one of ['gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl'].\n    device_id: Device for featurization. Supply a GPU id (e.g. 0 or 3) to use GPU. If no GPU with this id is found, use CPU\n    max_text_length: maximum number of tokens to consider. Default 1024\n    divergence_curve_discretization_size: Number of points to consider on the divergence curve. Default 25\n    mauve_scaling_factor: \"c\" from the paper. Default 5.\n    verbose: If True (default), print running time updates\n    seed: random seed to initialize k-means cluster assignments.\nReturns:\n    mauve: MAUVE score, a number between 0 and 1. Larger values indicate that P and Q are closer,\n    frontier_integral: Frontier Integral, a number between 0 and 1. Smaller values indicate that P and Q are closer,\n    divergence_curve: a numpy.ndarray of shape (m, 2); plot it with matplotlib to view the divergence curve,\n    p_hist: a discrete distribution, which is a quantized version of the text distribution p_text,\n    q_hist: same as above, but with q_text.\nExamples:\n\n    >>> # faiss segfaults in doctest for some reason, so the .compute call is not tested with doctest\n    >>> import datasets\n    >>> mauve = datasets.load_metric('mauve')\n    >>> predictions = [\"hello there\", \"general kenobi\"]\n    >>> references = [\"hello there\", \"general kenobi\"]\n    >>> out = mauve.compute(predictions=predictions, references=references) # doctest: +SKIP\n    >>> print(out.mauve) # doctest: +SKIP\n    1.0\n"






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION       ,	_KWARGS_DESCRIPTION   )
class 						A   (   datasets.Metric   ):





							def       snake_case__						(      self				:							List[Any]	)							->	Any:
								return datasets.MetricInfo(
								    description=_DESCRIPTION     ,		citation=_CITATION     ,		homepage='''https://github.com/krishnap25/mauve'''     ,		inputs_description=_KWARGS_DESCRIPTION     ,		features=datasets.Features(
								        {
								            '''predictions''': datasets.Value('''string'''     ,		id='''sequence'''	),
								            '''references''': datasets.Value('''string'''     ,		id='''sequence'''	),
								        }	)     ,		codebase_urls=['''https://github.com/krishnap25/mauve''']     ,		reference_urls=[
								        '''https://arxiv.org/abs/2102.01454''',
								        '''https://github.com/krishnap25/mauve''',
								    ]     ,		)






							def       snake_case__						(      self				:							str     ,		__a				:							List[Any]     ,		__a				:							Union[str, Any]     ,		__a				:							Union[str, Any]=None     ,		__a				:							Optional[Any]=None     ,		__a				:							Dict=None     ,		__a				:							Dict=None     ,		__a				:							int="auto"     ,		__a				:							List[Any]=-1     ,		__a				:							Any=0.9     ,		__a				:							Tuple=5     ,		__a				:							str=5_0_0     ,		__a				:							int="gpt2-large"     ,		__a				:							List[Any]=-1     ,		__a				:							Optional[int]=1_0_2_4     ,		__a				:							Tuple=2_5     ,		__a				:							List[str]=5     ,		__a				:							int=True     ,		__a				:							str=2_5     ,		)							->	List[Any]:
								__UpperCAmelCase							=			compute_mauve(
								    p_text=__a     ,		q_text=__a     ,		p_features=__a     ,		q_features=__a     ,		p_tokens=__a     ,		q_tokens=__a     ,		num_buckets=__a     ,		pca_max_data=__a     ,		kmeans_explained_var=__a     ,		kmeans_num_redo=__a     ,		kmeans_max_iter=__a     ,		featurize_model_name=__a     ,		device_id=__a     ,		max_text_length=__a     ,		divergence_curve_discretization_size=__a     ,		mauve_scaling_factor=__a     ,		verbose=__a     ,		seed=__a     ,		)
								return out

'''simple docstring'''


import ast
import os
import re
import shutil
import tempfile
import unittest
from unittest import mock

import torch

from accelerate.test_utils.examples import compare_against_test
from accelerate.test_utils.testing import TempDirTestCase, require_trackers, run_command, slow
from accelerate.utils import write_basic_config


# DataLoaders built from `test_samples/MRPC` for quick testing
# Should mock `{script_name}.get_dataloaders` via:
# @mock.patch("{script_name}.get_dataloaders", mocked_dataloaders)

__lowerCAmelCase	:       Optional[int]						       =	[
    "cross_validation.py",
    "gradient_accumulation.py",
    "local_sgd.py",
    "multi_process_metrics.py",
    "memory.py",
    "automatic_gradient_accumulation.py",
    "fsdp_with_peak_mem_tracking.py",
    "deepspeed_with_config_support.py",
    "megatron_lm_gpt_pretraining.py",
]






class 						A   (   unittest.TestCase   ):





							def       snake_case__						(      self				:							Any     ,		__a				:							str     ,		__a				:							bool     ,		__a				:							str = None     ,		__a				:							list = None	)							->	Tuple:
								__UpperCAmelCase							=			None
								__UpperCAmelCase							=			os.path.abspath(os.path.join('''examples'''     ,		'''by_feature'''	)	)
								__UpperCAmelCase							=			os.path.abspath('''examples'''	)
								for item in os.listdir(__a	):
									if item not in EXCLUDE_EXAMPLES:
										__UpperCAmelCase							=			os.path.join(__a     ,		__a	)
										if os.path.isfile(__a	) and ".py" in item_path:
											with self.subTest(
											    tested_script=__a     ,		feature_script=__a     ,		tested_section='''main()''' if parser_only else '''training_function()'''     ,		):
												__UpperCAmelCase							=			compare_against_test(
												    os.path.join(__a     ,		__a	)     ,		__a     ,		__a     ,		__a	)
												__UpperCAmelCase							=			'''\n'''.join(__a	)
												if special_strings is not None:
													for string in special_strings:
														__UpperCAmelCase							=			diff.replace(__a     ,		''''''	)
												self.assertEqual(__a     ,		''''''	)





							def       snake_case__						(      self				:							Optional[Any]	)							->	str:
								self.one_complete_example('''complete_nlp_example.py'''     ,		__a	)
								self.one_complete_example('''complete_nlp_example.py'''     ,		__a	)






							def       snake_case__						(      self				:							List[str]	)							->	Tuple:
								__UpperCAmelCase							=			os.path.abspath(os.path.join('''examples'''     ,		'''cv_example.py'''	)	)
								__UpperCAmelCase							=			[
								    ''' ''' * 1_6 + '''{\n\n''',
								    ''' ''' * 2_0 + '''"accuracy": eval_metric["accuracy"],\n\n''',
								    ''' ''' * 2_0 + '''"f1": eval_metric["f1"],\n\n''',
								    ''' ''' * 2_0 + '''"train_loss": total_loss.item() / len(train_dataloader),\n\n''',
								    ''' ''' * 2_0 + '''"epoch": epoch,\n\n''',
								    ''' ''' * 1_6 + '''},\n\n''',
								    ''' ''' * 1_6 + '''step=epoch,\n''',
								    ''' ''' * 1_2,
								    ''' ''' * 8 + '''for step, batch in enumerate(active_dataloader):\n''',
								]
								self.one_complete_example('''complete_cv_example.py'''     ,		__a     ,		__a     ,		__a	)
								self.one_complete_example('''complete_cv_example.py'''     ,		__a     ,		__a     ,		__a	)






@mock.patch.dict(os.environ       ,	{'''TESTING_MOCKED_DATALOADERS''': '''1'''}   )
class 						A   (   UpperCAmelCase   ):
							a_        =      False





							@classmethod
							def       snake_case__						(      cls				:							Tuple	)							->	str:
								super().setUpClass()
								__UpperCAmelCase							=			tempfile.mkdtemp()
								__UpperCAmelCase							=			os.path.join(cls._tmpdir     ,		'''default_config.yml'''	)

								write_basic_config(save_location=cls.configPath	)
								__UpperCAmelCase							=			['''accelerate''', '''launch''', '''--config_file''', cls.configPath]





							@classmethod
							def       snake_case__						(      cls				:							Dict	)							->	int:
								super().tearDownClass()
								shutil.rmtree(cls._tmpdir	)





							def       snake_case__						(      self				:							Tuple	)							->	Dict:
								__UpperCAmelCase							=			f"""
        examples/by_feature/checkpointing.py
        --checkpointing_steps epoch
        --output_dir {self.tmpdir}
        """.split()
								run_command(self._launch_args + testargs	)
								self.assertTrue(os.path.exists(os.path.join(self.tmpdir     ,		'''epoch_0'''	)	)	)





							def       snake_case__						(      self				:							str	)							->	int:
								__UpperCAmelCase							=			f"""
        examples/by_feature/checkpointing.py
        --checkpointing_steps 1
        --output_dir {self.tmpdir}
        """.split()
								__UpperCAmelCase							=			run_command(self._launch_args + testargs	)
								self.assertTrue(os.path.exists(os.path.join(self.tmpdir     ,		'''step_2'''	)	)	)





							def       snake_case__						(      self				:							Any	)							->	Any:
								__UpperCAmelCase							=			f"""
        examples/by_feature/checkpointing.py
        --resume_from_checkpoint {os.path.join(self.tmpdir     ,		'epoch_0'	)}
        """.split()
								__UpperCAmelCase							=			run_command(self._launch_args + testargs     ,		return_stdout=__a	)
								self.assertNotIn('''epoch 0:'''     ,		__a	)
								self.assertIn('''epoch 1:'''     ,		__a	)





							def       snake_case__						(      self				:							Tuple	)							->	Optional[int]:
								__UpperCAmelCase							=			f"""
        examples/by_feature/checkpointing.py
        --resume_from_checkpoint {os.path.join(self.tmpdir     ,		'step_2'	)}
        """.split()
								__UpperCAmelCase							=			run_command(self._launch_args + testargs     ,		return_stdout=__a	)
								if torch.cuda.is_available():
									__UpperCAmelCase							=			torch.cuda.device_count()
								else:
									__UpperCAmelCase							=			1
								if num_processes > 1:
									self.assertNotIn('''epoch 0:'''     ,		__a	)
									self.assertIn('''epoch 1:'''     ,		__a	)
								else:
									self.assertIn('''epoch 0:'''     ,		__a	)
									self.assertIn('''epoch 1:'''     ,		__a	)





							@slow
							def       snake_case__						(      self				:							Any	)							->	Optional[Any]:
								__UpperCAmelCase							=			'''
        examples/by_feature/cross_validation.py
        --num_folds 2
        '''.split()
								with mock.patch.dict(os.environ     ,		{'''TESTING_MOCKED_DATALOADERS''': '''0'''}	):
									__UpperCAmelCase							=			run_command(self._launch_args + testargs     ,		return_stdout=__a	)
									__UpperCAmelCase							=			re.findall('''({.+})'''     ,		__a	)
									__UpperCAmelCase							=			[r for r in results if '''accuracy''' in r][-1]
									__UpperCAmelCase							=			ast.literal_eval(__a	)
									self.assertGreaterEqual(results['''accuracy''']     ,		0.7_5	)





							def       snake_case__						(      self				:							Dict	)							->	int:
								__UpperCAmelCase							=			['''examples/by_feature/multi_process_metrics.py''']
								run_command(self._launch_args + testargs	)





							@require_trackers
							@mock.patch.dict(os.environ     ,		{'''WANDB_MODE''': '''offline'''}	)
							def       snake_case__						(      self				:							Optional[Any]	)							->	Optional[int]:
								with tempfile.TemporaryDirectory() as tmpdir:
									__UpperCAmelCase							=			f"""
            examples/by_feature/tracking.py
            --with_tracking
            --project_dir {tmpdir}
            """.split()
									run_command(self._launch_args + testargs	)
									self.assertTrue(os.path.exists(os.path.join(__a     ,		'''tracking'''	)	)	)





							def       snake_case__						(      self				:							Optional[int]	)							->	List[Any]:
								__UpperCAmelCase							=			['''examples/by_feature/gradient_accumulation.py''']
								run_command(self._launch_args + testargs	)






							def       snake_case__						(      self				:							Tuple	)							->	Optional[Any]:
								__UpperCAmelCase							=			['''examples/by_feature/local_sgd.py''']
								run_command(self._launch_args + testargs	)
