



from dataclasses import dataclass, field
from typing import Optional





@dataclass
class 		UpperCAmelCase						:
   SCREAMING_SNAKE_CASE__							=     field(
       default='''codeparrot/codeparrot'''       ,metadata={'''help''': '''Model name or path of model to be trained.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''./'''       ,metadata={'''help''': '''Save dir where model repo is cloned and models updates are saved to.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''codeparrot/codeparrot-clean-train'''       ,metadata={'''help''': '''Name or path of training dataset.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''codeparrot/codeparrot-clean-valid'''       ,metadata={'''help''': '''Name or path of validation dataset.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=2       ,metadata={'''help''': '''Batch size for training.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=2       ,metadata={'''help''': '''Batch size for evaluation.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=0.1       ,metadata={'''help''': '''Value of weight decay.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=1_0_0_0_0       ,metadata={'''help''': '''Size of buffer used to shuffle streaming dataset.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=2E-4       ,metadata={'''help''': '''Learning rate fo training.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default='''cosine'''       ,metadata={'''help''': '''Learning rate.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=7_5_0       ,metadata={'''help''': '''Number of warmup steps in the learning rate schedule.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=1_6       ,metadata={'''help''': '''Number of gradient accumulation steps.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=snake_case_       ,metadata={'''help''': '''Use gradient checkpointing to reduce memory footprint.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=5_0_0_0_0       ,metadata={'''help''': '''Maximum number of training steps.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=-1       ,metadata={'''help''': '''Maximum number of evaluation steps. If -1 the full dataset is evaluated.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=1_0_2_4       ,metadata={'''help''': '''Sequence lengths used for training.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=1       ,metadata={'''help''': '''Training seed.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=1_0_2_4       ,metadata={'''help''': '''Interval to save checkpoints. Measured as number of forward passes not training steps.'''}       ,)
   SCREAMING_SNAKE_CASE__							=     field(
       default=snake_case_       ,metadata={'''help''': '''States path if the training should continue from a checkpoint folder.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=snake_case_       ,metadata={'''help''': '''If True the data is pretokenized.'''}				)






@dataclass
class 		UpperCAmelCase						:
   SCREAMING_SNAKE_CASE__							=     field(
       default='''codeparrot/codeparrot'''       ,metadata={'''help''': '''Model name or path of model to be evaluated.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''codeparrot/codeparrot-clean-valid'''       ,metadata={'''help''': '''Name or path of validation dataset.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=2       ,metadata={'''help''': '''Batch size used for evaluation.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=-1       ,metadata={'''help''': '''Maximum number of evaluation steps. If -1 the full dataset is evaluated.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=1_0_2_4       ,metadata={'''help''': '''Length of sequences to be evaluated.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=1       ,metadata={'''help''': '''Random seed used for evaluation.'''}				)






@dataclass
class 		UpperCAmelCase						:
   SCREAMING_SNAKE_CASE__							=     field(
       default='''codeparrot/codeparrot'''       ,metadata={'''help''': '''Model name or path of model to be evaluated.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=snake_case_       ,metadata={'''help''': '''Number of workers used for code evaluation.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=snake_case_       ,metadata={'''help''': '''The number of human-eval tasks to run. If not included all tasks are evaluated.'''}       ,)
   SCREAMING_SNAKE_CASE__							=     field(
       default=snake_case_       ,metadata={'''help''': '''Sample from the language model\'s output distribution.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=0.2       ,metadata={'''help''': '''Sampling temperature used for generation.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=2_5_6       ,metadata={'''help''': '''Maximum number of newly generated tokens.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=0       ,metadata={'''help''': '''Top-k parameter used for generation.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=0.95       ,metadata={'''help''': '''Top-p parameter used for nucleus sampling.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=1_0       ,metadata={'''help''': '''Number of generations to run in parallel.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=2_0_0       ,metadata={'''help''': '''Number of completions to generate for each sample.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=1       ,metadata={'''help''': '''Random seed used for evaluation.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''eval_results.json'''       ,metadata={'''help''': '''Random seed used for evaluation.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''0'''       ,metadata={'''help''': '''Allow `code_eval` to execute Python code on machine'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=-1       ,metadata={
           '''help''': (
               '''Determine which device to run the `text-generation` Pipeline on. -1 is CPU and any zero or positive'''
               ''' number corresponds to which GPU device id to run on.'''
           )
       }       ,)






@dataclass
class 		UpperCAmelCase						:
   SCREAMING_SNAKE_CASE__							=     field(
       default=snake_case_       ,metadata={
           '''help''': '''The number of CPU cores to use for parallel preprocessing. Default uses the maximum available.'''
       }       ,)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''transformersbook/codeparrot'''       ,metadata={'''help''': '''Folder or name of dataset to process.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''codeparrot-clean'''       ,metadata={'''help''': '''Folder to save processed processed dataset.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=1_0_0_0_0_0       ,metadata={'''help''': '''Number of files to save per JSON output file.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default='''content'''       ,metadata={'''help''': '''Column containing text data to process.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=1_0_0_0       ,metadata={'''help''': '''Maximum line length in file, otherwise file is filtered.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=1_0_0       ,metadata={'''help''': '''Maximum mean line length in file, otherwise file is filtered.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=0.25       ,metadata={'''help''': '''Maximum fraction of non-alphanumeric characters, otherwise file is filtered.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=1.5       ,metadata={'''help''': '''Minimum character token ratio for the file, otherwise file is filtered.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=0.7       ,metadata={'''help''': '''Probability for filtering config, test and uncommon files.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''codeparrot/codeparrot'''       ,metadata={'''help''': '''Name or path to the tokenizer.'''}       ,)
   SCREAMING_SNAKE_CASE__							=     field(
       default=snake_case_       ,metadata={'''help''': '''If True, near-duplicate samples are removed.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=0.85       ,metadata={'''help''': '''Jaccard threshold for near-duplicate samples.'''}				)






@dataclass
class 		UpperCAmelCase						:
   SCREAMING_SNAKE_CASE__							=     field(
       default='''gpt2'''       ,metadata={'''help''': '''Base tokenizer to build new tokenizer from.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''transformersbook/codeparrot-train'''       ,metadata={'''help''': '''Dataset to train tokenizer on.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default='''content'''       ,metadata={'''help''': '''Column containing text data to process.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=2_0_0_0_0_0       ,metadata={'''help''': '''Number of examples to train tokenizer on.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default=3_2_7_6_8       ,metadata={'''help''': '''Number of examples to train the tokenizer on.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default='''codeparrot'''       ,metadata={'''help''': '''Name of new tokenizer.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=snake_case_       ,metadata={'''help''': '''Push saved tokenizer to the hub.'''}				)






@dataclass
class 		UpperCAmelCase						:
   SCREAMING_SNAKE_CASE__							=     field(
       default='''codeparrot/codeparrot'''       ,metadata={'''help''': '''Name or path to the tokenizer.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''codeparrot/codeparrot-clean-train'''       ,metadata={'''help''': '''Name or path to the dataset to pretokenize.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''tokenized-codeparrot-train'''       ,metadata={'''help''': '''Repo name of the pretokenized data.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=snake_case_       ,metadata={'''help''': '''Number of workers used for code evaluation.'''}				)






@dataclass
class 		UpperCAmelCase						:
   SCREAMING_SNAKE_CASE__							=     field(
       default='''gpt2-large'''       ,metadata={'''help''': '''Configuration to use for model initialization.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(
       default='''codeparrot/codeparrot'''       ,metadata={'''help''': '''Tokenizer attached to model.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default='''codeparrot'''       ,metadata={'''help''': '''Name of the created model.'''}				)
   SCREAMING_SNAKE_CASE__							=     field(default=snake_case_       ,metadata={'''help''': '''Push saved tokenizer to the hub.'''}				)




import functools
import gc
import inspect

import torch

from .imports import is_npu_available, is_xpu_available
def 							UpperCAmelCase__		(				*_SCREAMING_SNAKE_CASE   :	Tuple				)->List[Any]:
      if not isinstance(_SCREAMING_SNAKE_CASE     ,				_SCREAMING_SNAKE_CASE				):
            _lowerCAmelCase 			=  list(_SCREAMING_SNAKE_CASE				)
      for i in range(len(_SCREAMING_SNAKE_CASE				)				):
            _lowerCAmelCase 			=  None
      gc.collect()
      if is_xpu_available():
            torch.xpu.empty_cache()
      elif is_npu_available():
            torch.npu.empty_cache()
      else:
            torch.cuda.empty_cache()
      return objects
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Exception				)->bool:
      _lowerCAmelCase 			=  [
          '''CUDA out of memory.''',  # CUDA OOM
          '''cuDNN error: CUDNN_STATUS_NOT_SUPPORTED.''',  # CUDNN SNAFU
          '''DefaultCPUAllocator: can\'t allocate memory''',  # CPU OOM
      ]
      if isinstance(_SCREAMING_SNAKE_CASE     ,				_SCREAMING_SNAKE_CASE				) and len(exception.args				) == 1:
            return any(err in exception.args[0] for err in _statements				)
      return False


def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	callable = None     ,				_SCREAMING_SNAKE_CASE   :	int = 1_2_8				)->Optional[int]:
      if function is None:
            return functools.partial(_SCREAMING_SNAKE_CASE     ,				starting_batch_size=_SCREAMING_SNAKE_CASE				)

      _lowerCAmelCase 			=  starting_batch_size

      def decorator(*_SCREAMING_SNAKE_CASE   :	Optional[int]     ,				**_SCREAMING_SNAKE_CASE   :	Optional[Any]				):
            nonlocal batch_size
            gc.collect()
            if is_xpu_available():
                  torch.xpu.empty_cache()
            elif is_npu_available():
                  torch.npu.empty_cache()
            else:
                  torch.cuda.empty_cache()
            _lowerCAmelCase 			=  list(inspect.signature(_SCREAMING_SNAKE_CASE				).parameters.keys()				)
            # Guard against user error
            if len(_SCREAMING_SNAKE_CASE				) < (len(_SCREAMING_SNAKE_CASE				) + 1):
                  _lowerCAmelCase 			=  ''', '''.join([f'''{arg}={value}''' for arg, value in zip(params[1:]     ,				args[1:]				)]				)
                  raise TypeError(
                      f'''Batch size was passed into `{function.__name__}` as the first argument when called.'''
                      f'''Remove this as the decorator already does so: `{function.__name__}({arg_str})`'''				)
            while True:
                  if batch_size == 0:
                        raise RuntimeError('''No executable batch size found, reached zero.'''				)
                  try:
                        return function(_SCREAMING_SNAKE_CASE     ,				*_SCREAMING_SNAKE_CASE     ,				**_SCREAMING_SNAKE_CASE				)
                  except Exception as e:
                        if should_reduce_batch_size(_SCREAMING_SNAKE_CASE				):
                              gc.collect()
                              if is_xpu_available():
                                    torch.xpu.empty_cache()
                              elif is_npu_available():
                                    torch.npu.empty_cache()
                              else:
                                    torch.cuda.empty_cache()
                              batch_size //= 2
                        else:
                              raise

      return decorator