




import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase	:       Union[str, Any]     =	get_tests_dir('fixtures/test_sentencepiece.model')




@require_sentencepiece
class 					snake_case__							(_UpperCamelCase					, unittest.TestCase     ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_ :							Optional[int]											=      XLMProphetNetTokenizer
				SCREAMING_SNAKE_CASE_ :							Optional[int]											=      False
				SCREAMING_SNAKE_CASE_ :							str											=      True





				def 							__UpperCAmelCase							(   self       :  List[str]							)      ->  Dict:
							super().setUp()

							# We have a SentencePiece fixture for testing
							a					=    XLMProphetNetTokenizer(__lowerCamelCase      ,   keep_accents=__lowerCamelCase							)
							tokenizer.save_pretrained(self.tmpdirname							)





				def 							__UpperCAmelCase							(   self       :  Optional[int]							)      ->  Optional[Any]:
							a					=    "[PAD]"
							a					=    0

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase							)      ,   __lowerCamelCase							)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase							)      ,   __lowerCamelCase							)





				def 							__UpperCAmelCase							(   self       :  Union[str, Any]							)      ->  Union[str, Any]:
							a					=    list(self.get_tokenizer().get_vocab().keys()							)

							self.assertEqual(vocab_keys[0]      ,   "[PAD]"							)
							self.assertEqual(vocab_keys[1]      ,   "[CLS]"							)
							self.assertEqual(vocab_keys[-1]      ,   "j"							)
							self.assertEqual(len(__lowerCamelCase							)      ,   10_12							)





				def 							__UpperCAmelCase							(   self       :  Any							)      ->  Any:
							self.assertEqual(self.get_tokenizer().vocab_size      ,   10_12							)





				def 							__UpperCAmelCase							(   self       :  Optional[int]							)      ->  Dict:
							a					=    XLMProphetNetTokenizer(__lowerCamelCase      ,   keep_accents=__lowerCamelCase							)

							a					=    tokenizer.tokenize("This is a test"							)
							self.assertListEqual(__lowerCamelCase      ,   ["▁This", "▁is", "▁a", "▁t", "est"]							)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(__lowerCamelCase							)      ,   [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]      ,   )

							a					=    tokenizer.tokenize("I was born in 92000, and this is falsé."							)
							self.assertListEqual(
							    __lowerCamelCase      ,   [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "é",
							        ".",
							    ]      ,   )
							a					=    tokenizer.convert_tokens_to_ids(__lowerCamelCase							)
							self.assertListEqual(
							    __lowerCamelCase      ,   [
							        value + tokenizer.fairseq_offset
							        for value in [8, 21, 84, 55, 24, 19, 7, -9, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, -9, 4]
							    ]      ,   )

							a					=    tokenizer.convert_ids_to_tokens(__lowerCamelCase							)
							self.assertListEqual(
							    __lowerCamelCase      ,   [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "[UNK]",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "[UNK]",
							        ".",
							    ]      ,   )





				@cached_property
				def 							__UpperCAmelCase							(   self       :  Union[str, Any]							)      ->  Optional[Any]:
							return XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"							)





				@slow
				def 							__UpperCAmelCase							(   self       :  Tuple							)      ->  Union[str, Any]:
							a					=    "Hello World!"
							a					=    [3_53_89, 66_72, 49, 2]
							self.assertListEqual(__lowerCamelCase      ,   self.big_tokenizer.encode(__lowerCamelCase							)							)





				@slow
				def 							__UpperCAmelCase							(   self       :  Union[str, Any]							)      ->  Tuple:
							# fmt: off
							a					=    {"input_ids": [[1_10_73, 8_27_83, 18, 26, 8_27_83, 5_49, 5_15_40, 2_48, 1_72_09, 13_01, 2_17, 20, 21_51_86, 13_25, 1_47, 1_72_09, 13_01, 2_17, 20, 5_63_70, 53, 12_20_20, 20, 1_64_77, 27, 8_73_55, 45_48, 20, 47_28, 7_83_92, 17, 15_99_69, 18, 26, 2_44_91, 6_29, 15, 5_38, 2_27_04, 54_39, 15, 27_88, 2_44_91, 98_85, 15, 4_35_34, 6_05, 15, 8_14, 1_84_03, 3_32_00, 29, 15, 4_35_34, 2_44_58, 1_24_10, 1_11, 2_49_66, 8_36_69, 96_37, 14_40_68, 26, 8_50, 2_23_46, 27, 1_47, 2_49_66, 8_36_69, 8_34_90, 26, 3_91_13, 7_35, 27, 6_89, 6_56, 28_00, 13_39, 46_00, 53, 12_20_20, 11_57_85, 34, 8_16, 13_39, 4_68_87, 18, 1_47, 5_39_05, 19_51, 4_22_38, 4_11_70, 1_77_32, 8_34, 4_36, 15, 2_75_23, 9_87_33, 2_17, 1_47, 55_42, 49_81, 9_30, 1_73_47, 16, 2], [2_00_91, 6_29, 94, 8_27_86, 58, 4_90, 20, 15_28, 84, 5_39_05, 3_44, 8_05_92, 11_01_28, 1_88_22, 52_67, 13_06, 62, 15_25_37, 3_08, 79_97, 4_01, 12_44_27, 5_49, 3_54_42, 2_25, 1_09, 1_50_55, 2_57_48, 1_47, 71_19, 4_37_12, 34, 7_67, 13_53_66, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_92, 6_37_84, 11_94_66, 17, 14_78_08, 8_82_14, 18, 6_56, 81, 32, 32_96, 1_02_80, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=__lowerCamelCase      ,   model_name="microsoft/xprophetnet-large-wiki100-cased"      ,   revision="1acad1643ddd54a44df6a1b797ada8373685d90e"      ,   )












import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


__lowerCAmelCase	:       Union[str, Any]     =	['\nclass', '\ndef', '\n#', '\n@', '\nprint', '\nif']




class 					snake_case__							(_UpperCamelCase     ):
				"""simple docstring"""



				def __init__(   self       :  Any      ,   __lowerCamelCase       :  List[str]      ,   __lowerCamelCase       :  Union[str, Any]      ,   __lowerCamelCase       :  Union[str, Any]=None      ,   __lowerCamelCase       :  Any=1							)      ->  Union[str, Any]:
							a					=    tokenizer
							a					=    dataset
							a					=    len(__lowerCamelCase							) if n_tasks is None else n_tasks
							a					=    n_copies





				def __iter__(   self       :  Tuple							)      ->  str:
							a					=    []
							for task in range(self.n_tasks							):
										# without strip, the model generate commented codes ...
										prompts.append(self.tokenizer.eos_token + self.dataset[task]["prompt"].strip()							)
							a					=    self.tokenizer(__lowerCamelCase      ,   padding=__lowerCamelCase      ,   return_tensors="pt"							)
							for task in range(self.n_tasks							):
										for _ in range(self.n_copies							):
													yield {
													    "ids": outputs.input_ids[task],
													    "task_id": task,
													    "input_len": outputs.attention_mask[task].sum(),
													}




class 					snake_case__							(_UpperCamelCase     ):
				"""simple docstring"""



				def __init__(   self       :  int      ,   __lowerCamelCase       :  Dict      ,   __lowerCamelCase       :  Dict      ,   __lowerCamelCase       :  Tuple							)      ->  Optional[Any]:
							a					=    start_length
							a					=    eof_strings
							a					=    tokenizer





				def __call__(   self       :  int      ,   __lowerCamelCase       :  Dict      ,   __lowerCamelCase       :  List[str]      ,   **__lowerCamelCase       :  Optional[int]							)      ->  Optional[Any]:
							a					=    self.tokenizer.batch_decode(input_ids[:, self.start_length :]							)
							a					=    []
							for decoded_generation in decoded_generations:
										done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings							)							)
							return all(__lowerCamelCase							)




def     __magic_name__       (				A	:      List[Any]     ):






			'''simple docstring'''







			a					=    re.split("(%s)" % "|".join(A     ),						A     )
			# last string should be ""
			return "".join(string_list[:-2]     )




def     __magic_name__       (				A	:      Union[str, Any],						A	:      Optional[Any],						A	:      List[Any],						A	:      Optional[Any],						A	:      List[str],						A	:      List[Any]=20,						**A	:      Union[str, Any]     ):






			'''simple docstring'''







			a					=    defaultdict(A     )  # dict of list of generated tokens
			for step, batch in tqdm(enumerate(A     )     ):
						with torch.no_grad():
									a					=    batch["ids"].shape[-1]
									a					=    accelerator.unwrap_model(A     ).generate(
									    input_ids=batch["ids"][:, : batch["input_len"]],						num_return_sequences=A,						**A     )
									# each task is generated batch_size times
									a					=    batch["task_id"].repeat(A     )
									a					=    accelerator.pad_across_processes(
									    A,						dim=1,						pad_index=tokenizer.pad_token_id     )

									a					,	a					=    accelerator.gather((generated_tokens, generated_tasks)     )
									a					=    generated_tokens.cpu().numpy()
									a					=    generated_tasks.cpu().numpy()

									for task, generated_tokens in zip(A,						A     ):
												gen_token_dict[task].append(A     )

			a					=    [[] for _ in range(A     )]
			for task, generated_tokens in gen_token_dict.items():
						for s in generated_tokens:
									a					=    tokenizer.decode(A,						skip_special_tokens=A,						clean_up_tokenization_spaces=A     )
									code_gens[task].append(remove_last_block(A     )     )
			return code_gens




def     __magic_name__       (				):






			'''simple docstring'''







			a					=    HfArgumentParser(A     )
			a					=    parser.parse_args()

			transformers.logging.set_verbosity_error()
			# enables code execution in code_eval metric
			a					=    args.HF_ALLOW_CODE_EVAL
			# make sure tokenizer plays nice with multiprocessing
			a					=    "false"

			if args.num_workers is None:
						a					=    multiprocessing.cpu_count()

			# Use dataset load to feed to accelerate
			a					=    Accelerator()
			set_seed(args.seed,						device_specific=A     )

			# Load model and tokenizer
			a					=    AutoTokenizer.from_pretrained(args.model_ckpt     )
			a					=    tokenizer.eos_token
			a					=    AutoModelForCausalLM.from_pretrained(args.model_ckpt     )

			# Generation settings
			a					=    {
			    "do_sample": args.do_sample,
			    "temperature": args.temperature,
			    "max_new_tokens": args.max_new_tokens,
			    "top_p": args.top_p,
			    "top_k": args.top_k,
			    "stopping_criteria": StoppingCriteriaList([EndOfFunctionCriteria(0,						A,						A     )]     ),
			}

			# Load evaluation dataset and metric
			a					=    load_dataset("openai_humaneval"     )
			a					=    load_metric("code_eval"     )

			a					=    args.num_tasks if args.num_tasks is not None else len(human_eval["test"]     )
			a					=    args.n_samples // args.batch_size

			a					=    TokenizedDataset(A,						human_eval["test"],						n_copies=A,						n_tasks=A     )
			# do not confuse args.batch_size, which is actually the num_return_sequences
			a					=    DataLoader(A,						batch_size=1     )

			# Run a quick test to see if code evaluation is enabled
			try:
						a					=    code_eval_metric.compute(references=[""],						predictions=[[""]]     )
			except ValueError as exception:
						print(
						    "Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"
						    " flag to enable code evaluation."     )
						raise exception

			a					,	a					=    accelerator.prepare(A,						A     )

			a					=    complete_code(
			    A,						A,						A,						A,						n_tasks=A,						batch_size=args.batch_size,						**A,						)

			if accelerator.is_main_process:
						a					=    []

						for task in tqdm(range(A     )     ):
									a					=    human_eval["test"][task]["test"]
									a					=    F"""check({human_eval["test"][task]["entry_point"]})"""
									references.append("\n" + test_func + "\n" + entry_point     )

						# Evaluate completions with "code_eval" metric
						a					,	a					=    code_eval_metric.compute(
						    references=A,						predictions=A,						num_workers=args.num_workers     )
						print(F"""Results: {pass_at_k}"""     )

						# Save results to json file
						with open(args.output_file,						"w"     ) as fp:
									json.dump(A,						A     )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
		main()






