

from dataclasses import dataclass, field
from typing import Optional







@dataclass
class 	lowercase    :


		"""simple docstring"""




		a__					       =  field(
		    default="codeparrot/codeparrot"    ,							metadata={"help": "Model name or path of model to be trained."}							)
		a__					       =  field(
		    default="./"    ,							metadata={"help": "Save dir where model repo is cloned and models updates are saved to."}							)
		a__					       =  field(
		    default="codeparrot/codeparrot-clean-train"    ,							metadata={"help": "Name or path of training dataset."}							)
		a__					       =  field(
		    default="codeparrot/codeparrot-clean-valid"    ,							metadata={"help": "Name or path of validation dataset."}							)
		a__					       =  field(default=2    ,							metadata={"help": "Batch size for training."}							)
		a__					       =  field(default=2    ,							metadata={"help": "Batch size for evaluation."}							)
		a__					       =  field(default=0.1    ,							metadata={"help": "Value of weight decay."}							)
		a__					       =  field(
		    default=1_0_0_0_0    ,							metadata={"help": "Size of buffer used to shuffle streaming dataset."}							)
		a__					       =  field(default=2e-4    ,							metadata={"help": "Learning rate fo training."}							)
		a__					       =  field(default="cosine"    ,							metadata={"help": "Learning rate."}							)
		a__					       =  field(
		    default=7_5_0    ,							metadata={"help": "Number of warmup steps in the learning rate schedule."}							)
		a__					       =  field(
		    default=1_6    ,							metadata={"help": "Number of gradient accumulation steps."}							)
		a__					       =  field(
		    default=_lowercase    ,							metadata={"help": "Use gradient checkpointing to reduce memory footprint."}							)
		a__					       =  field(default=5_0_0_0_0    ,							metadata={"help": "Maximum number of training steps."}							)
		a__					       =  field(
		    default=-1    ,							metadata={"help": "Maximum number of evaluation steps. If -1 the full dataset is evaluated."}							)
		a__					       =  field(default=1_0_2_4    ,							metadata={"help": "Sequence lengths used for training."}							)
		a__					       =  field(default=1    ,							metadata={"help": "Training seed."}							)
		a__					       =  field(
		    default=1_0_2_4    ,							metadata={"help": "Interval to save checkpoints. Measured as number of forward passes not training steps."}    ,							)
		a__					       =  field(
		    default=_lowercase    ,							metadata={"help": "States path if the training should continue from a checkpoint folder."}							)
		a__					       =  field(default=_lowercase    ,							metadata={"help": "If True the data is pretokenized."}							)







@dataclass
class 	lowercase    :


		"""simple docstring"""




		a__					       =  field(
		    default="codeparrot/codeparrot"    ,							metadata={"help": "Model name or path of model to be evaluated."}							)
		a__					       =  field(
		    default="codeparrot/codeparrot-clean-valid"    ,							metadata={"help": "Name or path of validation dataset."}							)
		a__					       =  field(default=2    ,							metadata={"help": "Batch size used for evaluation."}							)
		a__					       =  field(
		    default=-1    ,							metadata={"help": "Maximum number of evaluation steps. If -1 the full dataset is evaluated."}							)
		a__					       =  field(default=1_0_2_4    ,							metadata={"help": "Length of sequences to be evaluated."}							)
		a__					       =  field(default=1    ,							metadata={"help": "Random seed used for evaluation."}							)







@dataclass
class 	lowercase    :


		"""simple docstring"""




		a__					       =  field(
		    default="codeparrot/codeparrot"    ,							metadata={"help": "Model name or path of model to be evaluated."}							)
		a__					       =  field(default=_lowercase    ,							metadata={"help": "Number of workers used for code evaluation."}							)
		a__					       =  field(
		    default=_lowercase    ,							metadata={"help": "The number of human-eval tasks to run. If not included all tasks are evaluated."}    ,							)
		a__					       =  field(
		    default=_lowercase    ,							metadata={"help": "Sample from the language model's output distribution."}							)
		a__					       =  field(default=0.2    ,							metadata={"help": "Sampling temperature used for generation."}							)
		a__					       =  field(default=2_5_6    ,							metadata={"help": "Maximum number of newly generated tokens."}							)
		a__					       =  field(default=0    ,							metadata={"help": "Top-k parameter used for generation."}							)
		a__					       =  field(default=0.95    ,							metadata={"help": "Top-p parameter used for nucleus sampling."}							)
		a__					       =  field(default=1_0    ,							metadata={"help": "Number of generations to run in parallel."}							)
		a__					       =  field(
		    default=2_0_0    ,							metadata={"help": "Number of completions to generate for each sample."}							)
		a__					       =  field(default=1    ,							metadata={"help": "Random seed used for evaluation."}							)
		a__					       =  field(
		    default="eval_results.json"    ,							metadata={"help": "Random seed used for evaluation."}							)
		a__					       =  field(
		    default="0"    ,							metadata={"help": "Allow `code_eval` to execute Python code on machine"}							)
		a__					       =  field(
		    default=-1    ,							metadata={
		        "help": (
		            "Determine which device to run the `text-generation` Pipeline on. -1 is CPU and any zero or positive"
		            " number corresponds to which GPU device id to run on."
		        )
		    }    ,							)







@dataclass
class 	lowercase    :


		"""simple docstring"""




		a__					       =  field(
		    default=_lowercase    ,							metadata={
		        "help": "The number of CPU cores to use for parallel preprocessing. Default uses the maximum available."
		    }    ,							)
		a__					       =  field(
		    default="transformersbook/codeparrot"    ,							metadata={"help": "Folder or name of dataset to process."}							)
		a__					       =  field(
		    default="codeparrot-clean"    ,							metadata={"help": "Folder to save processed processed dataset."}							)
		a__					       =  field(
		    default=1_0_0_0_0_0    ,							metadata={"help": "Number of files to save per JSON output file."}							)
		a__					       =  field(default="content"    ,							metadata={"help": "Column containing text data to process."}							)
		a__					       =  field(
		    default=1_0_0_0    ,							metadata={"help": "Maximum line length in file, otherwise file is filtered."}							)
		a__					       =  field(
		    default=1_0_0    ,							metadata={"help": "Maximum mean line length in file, otherwise file is filtered."}							)
		a__					       =  field(
		    default=0.25    ,							metadata={"help": "Maximum fraction of non-alphanumeric characters, otherwise file is filtered."}							)
		a__					       =  field(
		    default=1.5    ,							metadata={"help": "Minimum character token ratio for the file, otherwise file is filtered."}							)
		a__					       =  field(
		    default=0.7    ,							metadata={"help": "Probability for filtering config, test and uncommon files."}							)
		a__					       =  field(
		    default="codeparrot/codeparrot"    ,							metadata={"help": "Name or path to the tokenizer."}    ,							)
		a__					       =  field(
		    default=_lowercase    ,							metadata={"help": "If True, near-duplicate samples are removed."}							)
		a__					       =  field(
		    default=0.85    ,							metadata={"help": "Jaccard threshold for near-duplicate samples."}							)







@dataclass
class 	lowercase    :


		"""simple docstring"""




		a__					       =  field(
		    default="gpt2"    ,							metadata={"help": "Base tokenizer to build new tokenizer from."}							)
		a__					       =  field(
		    default="transformersbook/codeparrot-train"    ,							metadata={"help": "Dataset to train tokenizer on."}							)
		a__					       =  field(default="content"    ,							metadata={"help": "Column containing text data to process."}							)
		a__					       =  field(default=2_0_0_0_0_0    ,							metadata={"help": "Number of examples to train tokenizer on."}							)
		a__					       =  field(
		    default=3_2_7_6_8    ,							metadata={"help": "Number of examples to train the tokenizer on."}							)
		a__					       =  field(default="codeparrot"    ,							metadata={"help": "Name of new tokenizer."}							)
		a__					       =  field(default=_lowercase    ,							metadata={"help": "Push saved tokenizer to the hub."}							)







@dataclass
class 	lowercase    :


		"""simple docstring"""




		a__					       =  field(
		    default="codeparrot/codeparrot"    ,							metadata={"help": "Name or path to the tokenizer."}							)
		a__					       =  field(
		    default="codeparrot/codeparrot-clean-train"    ,							metadata={"help": "Name or path to the dataset to pretokenize."}							)
		a__					       =  field(
		    default="tokenized-codeparrot-train"    ,							metadata={"help": "Repo name of the pretokenized data."}							)
		a__					       =  field(default=_lowercase    ,							metadata={"help": "Number of workers used for code evaluation."}							)







@dataclass
class 	lowercase    :


		"""simple docstring"""




		a__					       =  field(
		    default="gpt2-large"    ,							metadata={"help": "Configuration to use for model initialization."}							)
		a__					       =  field(
		    default="codeparrot/codeparrot"    ,							metadata={"help": "Tokenizer attached to model."}							)
		a__					       =  field(default="codeparrot"    ,							metadata={"help": "Name of the created model."}							)
		a__					       =  field(default=_lowercase    ,							metadata={"help": "Push saved tokenizer to the hub."}							)







# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import doctest
import sys
import warnings
from os.path import abspath, dirname, join

import _pytest

from transformers.testing_utils import HfDoctestModule, HfDocTestParser


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
lowerCAmelCase__								=		abspath(join(dirname(__file__), """src"""))
sys.path.insert(1, git_repo_path)

# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action="""ignore""", category=FutureWarning)






def 					lowerCamelCase_     (							UpperCAmelCase_							:     Any	)						->				Union[str, Any]:




	'''simple docstring'''




	config.addinivalue_line(
	    'markers'							,       'is_pt_tf_cross_test: mark test to run only when PT and TF interactions are tested'	)
	config.addinivalue_line(
	    'markers'							,       'is_pt_flax_cross_test: mark test to run only when PT and FLAX interactions are tested'	)
	config.addinivalue_line('markers'							,       'is_pipeline_test: mark test to run only when pipelines are tested'	)
	config.addinivalue_line('markers'							,       'is_staging_test: mark test to run only in the staging environment'	)
	config.addinivalue_line('markers'							,       'accelerate_tests: mark test that require accelerate'	)
	config.addinivalue_line('markers'							,       'tool_tests: mark the tool tests that are run on their specific schedule'	)






def 					lowerCamelCase_     (							UpperCAmelCase_							:     Optional[int]	)						->				List[Any]:




	'''simple docstring'''




	from transformers.testing_utils import pytest_addoption_shared

	pytest_addoption_shared(UpperCAmelCase_	)






def 					lowerCamelCase_     (							UpperCAmelCase_							:     Any	)						->				List[Any]:




	'''simple docstring'''




	from transformers.testing_utils import pytest_terminal_summary_main

	_UpperCamelCase   :		str			=						terminalreporter.config.getoption('--make-reports'	)
	if make_reports:
		pytest_terminal_summary_main(UpperCAmelCase_							,       id=UpperCAmelCase_	)






def 					lowerCamelCase_     (							UpperCAmelCase_							:     Dict							,       UpperCAmelCase_							:     Dict	)						->				Tuple:




	'''simple docstring'''




	if exitstatus == 5:
		_UpperCamelCase   :		List[Any]			=						0


# Doctest custom flag to ignore output.
lowerCAmelCase__								=		doctest.register_optionflag("""IGNORE_RESULT""")

lowerCAmelCase__								=		doctest.OutputChecker







class 	lowercase    (				_lowercase							):


		"""simple docstring"""




		def 		A__					(				self	,       __snake_case	,       __snake_case	,       __snake_case):
			if IGNORE_RESULT & optionflags:
				return True
			return OutputChecker.check_output(self	,       __snake_case	,       __snake_case	,       __snake_case)

lowerCAmelCase__								=		CustomOutputChecker
lowerCAmelCase__								=		HfDoctestModule







lowerCAmelCase__								=		HfDocTestParser




