






import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


A__			:    str         =				['\nclass', '\ndef', '\n#', '\n@', '\nprint', '\nif']
class    _UpperCAmelCase      ( __SCREAMING_SNAKE_CASE			):

    """simple docstring"""




    def __init__(  self		:					List[Any],						lowerCamelCase		:					Optional[int],						lowerCamelCase		:					List[Any],						lowerCamelCase		:					Any=None,						lowerCamelCase		:					int=1						):

        '''simple docstring'''
        lowercase__     				=    tokenizer
        lowercase__     				=    dataset
        lowercase__     				=    len(__snake_case						) if n_tasks is None else n_tasks
        lowercase__     				=    n_copies




    def __iter__(  self		:					Tuple						):

        '''simple docstring'''
        lowercase__     				=    []
        for task in range(self.n_tasks						):
            # without strip, the model generate commented codes ...
            prompts.append(self.tokenizer.eos_token + self.dataset[task]['''prompt'''].strip()						)
        lowercase__     				=    self.tokenizer(__snake_case,						padding=__snake_case,						return_tensors='''pt'''						)
        for task in range(self.n_tasks						):
            for _ in range(self.n_copies						):
                yield {
                    "ids": outputs.input_ids[task],
                    "task_id": task,
                    "input_len": outputs.attention_mask[task].sum(),
                }



class    _UpperCAmelCase      ( __SCREAMING_SNAKE_CASE			):

    """simple docstring"""




    def __init__(  self		:					Optional[int],						lowerCamelCase		:					Union[str, Any],						lowerCamelCase		:					str,						lowerCamelCase		:					int						):

        '''simple docstring'''
        lowercase__     				=    start_length
        lowercase__     				=    eof_strings
        lowercase__     				=    tokenizer




    def __call__(  self		:					Any,						lowerCamelCase		:					str,						lowerCamelCase		:					Dict,						**lowerCamelCase		:					List[str]						):

        '''simple docstring'''
        lowercase__     				=    self.tokenizer.batch_decode(input_ids[:, self.start_length :]						)
        lowercase__     				=    []
        for decoded_generation in decoded_generations:
            done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings						)						)
        return all(__snake_case						)






def 						a    (		lowerCamelCase_       ):






    '''simple docstring'''
    lowercase__     				=    re.split('''(%s)''' % '''|'''.join(a_       )	,     a_       )
    # last string should be ""
    return "".join(string_list[:-2]       )






def 						a    (		lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_=20	,     **lowerCamelCase_       ):






    '''simple docstring'''
    lowercase__     				=    defaultdict(a_       )  # dict of list of generated tokens
    for step, batch in tqdm(enumerate(a_       )       ):
        with torch.no_grad():
            lowercase__     				=    batch['''ids'''].shape[-1]
            lowercase__     				=    accelerator.unwrap_model(a_       ).generate(
                input_ids=batch['''ids'''][:, : batch['''input_len''']]	,     num_return_sequences=a_	,     **a_       )
            # each task is generated batch_size times
            lowercase__     				=    batch['''task_id'''].repeat(a_       )
            lowercase__     				=    accelerator.pad_across_processes(
                a_	,     dim=1	,     pad_index=tokenizer.pad_token_id       )

            lowercase__     				=    accelerator.gather((generated_tokens, generated_tasks)       )
            lowercase__     				=    generated_tokens.cpu().numpy()
            lowercase__     				=    generated_tasks.cpu().numpy()

            for task, generated_tokens in zip(a_	,     a_       ):
                gen_token_dict[task].append(a_       )

    lowercase__     				=    [[] for _ in range(a_       )]
    for task, generated_tokens in gen_token_dict.items():
        for s in generated_tokens:
            lowercase__     				=    tokenizer.decode(a_	,     skip_special_tokens=a_	,     clean_up_tokenization_spaces=a_       )
            code_gens[task].append(remove_last_block(a_       )       )
    return code_gens






def 						a    (		):






    '''simple docstring'''
    # Setup configuration
    lowercase__     				=    HfArgumentParser(a_       )
    lowercase__     				=    parser.parse_args()

    transformers.logging.set_verbosity_error()
    # enables code execution in code_eval metric
    lowercase__     				=    args.HF_ALLOW_CODE_EVAL
    # make sure tokenizer plays nice with multiprocessing
    lowercase__     				=    '''false'''

    if args.num_workers is None:
        lowercase__     				=    multiprocessing.cpu_count()

    # Use dataset load to feed to accelerate
    lowercase__     				=    Accelerator()
    set_seed(args.seed	,     device_specific=a_       )

    # Load model and tokenizer
    lowercase__     				=    AutoTokenizer.from_pretrained(args.model_ckpt       )
    lowercase__     				=    tokenizer.eos_token
    lowercase__     				=    AutoModelForCausalLM.from_pretrained(args.model_ckpt       )

    # Generation settings
    lowercase__     				=    {
        '''do_sample''': args.do_sample,
        '''temperature''': args.temperature,
        '''max_new_tokens''': args.max_new_tokens,
        '''top_p''': args.top_p,
        '''top_k''': args.top_k,
        '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0	,     a_	,     a_       )]       ),
    }

    # Load evaluation dataset and metric
    lowercase__     				=    load_dataset('''openai_humaneval'''       )
    lowercase__     				=    load_metric('''code_eval'''       )

    lowercase__     				=    args.num_tasks if args.num_tasks is not None else len(human_eval['''test''']       )
    lowercase__     				=    args.n_samples // args.batch_size

    lowercase__     				=    TokenizedDataset(a_	,     human_eval['''test''']	,     n_copies=a_	,     n_tasks=a_       )
    # do not confuse args.batch_size, which is actually the num_return_sequences
    lowercase__     				=    DataLoader(a_	,     batch_size=1       )

    # Run a quick test to see if code evaluation is enabled
    try:
        lowercase__     				=    code_eval_metric.compute(references=['''''']	,     predictions=[['''''']]       )
    except ValueError as exception:
        print(
            '''Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'''
            ''' flag to enable code evaluation.'''       )
        raise exception

    lowercase__     				=    accelerator.prepare(a_	,     a_       )

    lowercase__     				=    complete_code(
        a_	,     a_	,     a_	,     a_	,     n_tasks=a_	,     batch_size=args.batch_size	,     **a_	,     )

    if accelerator.is_main_process:
        lowercase__     				=    []

        for task in tqdm(range(a_       )       ):
            lowercase__     				=    human_eval['''test'''][task]['''test''']
            lowercase__     				=    F"""check({human_eval['test'][task]['entry_point']})"""
            references.append('''\n''' + test_func + '''\n''' + entry_point       )

        # Evaluate completions with "code_eval" metric
        lowercase__     				=    code_eval_metric.compute(
            references=a_	,     predictions=a_	,     num_workers=args.num_workers       )
        print(F"""Results: {pass_at_k}"""       )

        # Save results to json file
        with open(args.output_file	,     '''w'''       ) as fp:
            json.dump(a_	,     a_       )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
       main()












def 						a    (		lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_       ):






				'''simple docstring'''

				if exponent == 1:
								return base
				if exponent % 2 == 0:
								lowercase__     				=    _modexpt(lowerCamelCase_	,     exponent // 2	,     lowerCamelCase_       ) % modulo_value
								return (x * x) % modulo_value
				else:
								return (base * _modexpt(lowerCamelCase_	,     exponent - 1	,     lowerCamelCase_       )) % modulo_value






def 						a    (		lowerCamelCase_ = 1777	,     lowerCamelCase_ = 1855	,     lowerCamelCase_ = 8       ):






				'''simple docstring'''
				lowercase__     				=    base
				for _ in range(1	,     lowerCamelCase_       ):
								lowercase__     				=    _modexpt(lowerCamelCase_	,     lowerCamelCase_	,     10**digits       )

				return result


if __name__ == "__main__":
							print(F"{solution() = }")




