






'''simple docstring'''




import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


__lowerCamelCase   :					int   							=					['''\nclass''', '''\ndef''', '''\n#''', '''\n@''', '''\nprint''', '''\nif''']







class    A_	(a_							):
							"""simple docstring"""




							def __init__(		self			:Any       ,							lowerCAmelCase__			:Tuple       ,							lowerCAmelCase__			:Union[str, Any]       ,							lowerCAmelCase__			:Any=None       ,							lowerCAmelCase__			:Tuple=1				)				->  Union[str, Any]:


										'''simple docstring'''




										snake_case_				:					Union[str, Any]   =       tokenizer
										snake_case_				:					Union[str, Any]   =       dataset
										snake_case_				:					str   =       len(lowerCAmelCase__				) if n_tasks is None else n_tasks
										snake_case_				:					Any   =       n_copies






							def __iter__(		self			:Union[str, Any]				)				->  str:


										'''simple docstring'''




										snake_case_				:					Tuple   =       []
										for task in range(self.n_tasks				):
													# without strip, the model generate commented codes ...
													prompts.append(self.tokenizer.eos_token + self.dataset[task]["prompt"].strip()				)
										snake_case_				:					List[Any]   =       self.tokenizer(lowerCAmelCase__       ,							padding=lowerCAmelCase__       ,							return_tensors="pt"				)
										for task in range(self.n_tasks				):
													for _ in range(self.n_copies				):
																yield {
																    "ids": outputs.input_ids[task],
																    "task_id": task,
																    "input_len": outputs.attention_mask[task].sum(),
																}







class    A_	(a_							):
							"""simple docstring"""




							def __init__(		self			:Optional[int]       ,							lowerCAmelCase__			:List[Any]       ,							lowerCAmelCase__			:List[Any]       ,							lowerCAmelCase__			:Union[str, Any]				)				->  Union[str, Any]:


										'''simple docstring'''




										snake_case_				:					int   =       start_length
										snake_case_				:					str   =       eof_strings
										snake_case_				:					Optional[int]   =       tokenizer






							def __call__(		self			:Optional[Any]       ,							lowerCAmelCase__			:List[Any]       ,							lowerCAmelCase__			:List[str]       ,							**lowerCAmelCase__			:Optional[Any]				)				->  List[str]:


										'''simple docstring'''




										snake_case_				:					List[str]   =       self.tokenizer.batch_decode(input_ids[:, self.start_length :]				)
										snake_case_				:					int   =       []
										for decoded_generation in decoded_generations:
													done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings				)				)
										return all(lowerCAmelCase__				)

def 	__UpperCAmelCase       ( __magic_name__						)->    int:



			"""simple docstring"""



			snake_case_				:					List[Any]   =       re.split("(%s)" % "|".join(__magic_name__						)     ,__magic_name__						)
			# last string should be ""
			return "".join(string_list[:-2]						)

def 	__UpperCAmelCase       ( __magic_name__     ,__magic_name__     ,__magic_name__     ,__magic_name__     ,__magic_name__     ,__magic_name__=20     ,**__magic_name__						)->    Dict:



			"""simple docstring"""



			snake_case_				:					Dict   =       defaultdict(__magic_name__						)  # dict of list of generated tokens
			for step, batch in tqdm(enumerate(__magic_name__						)						):
						with torch.no_grad():
									snake_case_				:					List[str]   =       batch["ids"].shape[-1]
									snake_case_				:					Any   =       accelerator.unwrap_model(__magic_name__						).generate(
									    input_ids=batch["ids"][:, : batch["input_len"]]     ,num_return_sequences=__magic_name__     ,**__magic_name__						)
									# each task is generated batch_size times
									snake_case_				:					Tuple   =       batch["task_id"].repeat(__magic_name__						)
									snake_case_				:					Any   =       accelerator.pad_across_processes(
									    __magic_name__     ,dim=1     ,pad_index=tokenizer.pad_token_id						)

									snake_case_,	snake_case_				:					Dict   =       accelerator.gather((generated_tokens, generated_tasks)						)
									snake_case_				:					List[Any]   =       generated_tokens.cpu().numpy()
									snake_case_				:					Any   =       generated_tasks.cpu().numpy()

									for task, generated_tokens in zip(__magic_name__     ,__magic_name__						):
												gen_token_dict[task].append(__magic_name__						)

			snake_case_				:					List[str]   =       [[] for _ in range(__magic_name__						)]
			for task, generated_tokens in gen_token_dict.items():
						for s in generated_tokens:
									snake_case_				:					List[Any]   =       tokenizer.decode(__magic_name__     ,skip_special_tokens=__magic_name__     ,clean_up_tokenization_spaces=__magic_name__						)
									code_gens[task].append(remove_last_block(__magic_name__						)						)
			return code_gens

def 	__UpperCAmelCase       ( )->    Optional[int]:



			"""simple docstring"""



			snake_case_				:					Optional[Any]   =       HfArgumentParser(__magic_name__						)
			snake_case_				:					Union[str, Any]   =       parser.parse_args()

			transformers.logging.set_verbosity_error()
			# enables code execution in code_eval metric
			snake_case_				:					int   =       args.HF_ALLOW_CODE_EVAL
			# make sure tokenizer plays nice with multiprocessing
			snake_case_				:					Optional[Any]   =       "false"

			if args.num_workers is None:
						snake_case_				:					Tuple   =       multiprocessing.cpu_count()

			# Use dataset load to feed to accelerate
			snake_case_				:					Any   =       Accelerator()
			set_seed(args.seed     ,device_specific=__magic_name__						)

			# Load model and tokenizer
			snake_case_				:					List[Any]   =       AutoTokenizer.from_pretrained(args.model_ckpt						)
			snake_case_				:					Optional[int]   =       tokenizer.eos_token
			snake_case_				:					Union[str, Any]   =       AutoModelForCausalLM.from_pretrained(args.model_ckpt						)

			# Generation settings
			snake_case_				:					List[str]   =       {
			    "do_sample": args.do_sample,
			    "temperature": args.temperature,
			    "max_new_tokens": args.max_new_tokens,
			    "top_p": args.top_p,
			    "top_k": args.top_k,
			    "stopping_criteria": StoppingCriteriaList([EndOfFunctionCriteria(0     ,__magic_name__     ,__magic_name__						)]						),
			}

			# Load evaluation dataset and metric
			snake_case_				:					Optional[int]   =       load_dataset("openai_humaneval"						)
			snake_case_				:					int   =       load_metric("code_eval"						)

			snake_case_				:					Dict   =       args.num_tasks if args.num_tasks is not None else len(human_eval["test"]						)
			snake_case_				:					int   =       args.n_samples // args.batch_size

			snake_case_				:					Any   =       TokenizedDataset(__magic_name__     ,human_eval["test"]     ,n_copies=__magic_name__     ,n_tasks=__magic_name__						)
			# do not confuse args.batch_size, which is actually the num_return_sequences
			snake_case_				:					Any   =       DataLoader(__magic_name__     ,batch_size=1						)

			# Run a quick test to see if code evaluation is enabled
			try:
						snake_case_				:					List[Any]   =       code_eval_metric.compute(references=[""]     ,predictions=[[""]]						)
			except ValueError as exception:
						print(
						    "Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"
						    " flag to enable code evaluation."						)
						raise exception

			snake_case_,	snake_case_				:					str   =       accelerator.prepare(__magic_name__     ,__magic_name__						)

			snake_case_				:					str   =       complete_code(
			    __magic_name__     ,__magic_name__     ,__magic_name__     ,__magic_name__     ,n_tasks=__magic_name__     ,batch_size=args.batch_size     ,**__magic_name__     ,)

			if accelerator.is_main_process:
						snake_case_				:					List[Any]   =       []

						for task in tqdm(range(__magic_name__						)						):
									snake_case_				:					Dict   =       human_eval["test"][task]["test"]
									snake_case_				:					Dict   =       F'''check({human_eval['test'][task]['entry_point']})'''
									references.append("\n" + test_func + "\n" + entry_point						)

						# Evaluate completions with "code_eval" metric
						snake_case_,	snake_case_				:					Tuple   =       code_eval_metric.compute(
						    references=__magic_name__     ,predictions=__magic_name__     ,num_workers=args.num_workers						)
						print(F'''Results: {pass_at_k}'''						)

						# Save results to json file
						with open(args.output_file     ,"w"						) as fp:
									json.dump(__magic_name__     ,__magic_name__						)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
		main()












'''simple docstring'''




import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    Pipeline,
    ZeroShotClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
__lowerCamelCase   :					List[str]   							=					{'''LayoutLMv2Config''', '''LayoutLMv3Config'''}







@is_pipeline_test
class    A_	(unittest.TestCase							):
							"""simple docstring"""




							a__   		=	MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
							a__   		=	TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

							if model_mapping is not None:
														a__   		=	{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
							if tf_model_mapping is not None:
														a__   		=	{
														    config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
														}

							def 							_A	(		self			:Tuple       ,							lowerCAmelCase__			:Tuple       ,							lowerCAmelCase__			:Union[str, Any]       ,							lowerCAmelCase__			:Dict				)				->  Any:


										'''simple docstring'''




										snake_case_				:					Optional[Any]   =       ZeroShotClassificationPipeline(
										    model=lowerCAmelCase__       ,							tokenizer=lowerCAmelCase__       ,							candidate_labels=["polics", "health"]				)
										return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]

							def 							_A	(		self			:List[str]       ,							lowerCAmelCase__			:Optional[int]       ,							lowerCAmelCase__			:Any				)				->  Optional[Any]:


										'''simple docstring'''




										snake_case_				:					Tuple   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels="politics"				)
										self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

										# No kwarg
										snake_case_				:					List[Any]   =       classifier("Who are you voting for in 2020?"       ,							["politics"]				)
										self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

										snake_case_				:					Dict   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels=["politics"]				)
										self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

										snake_case_				:					int   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels="politics, public health"				)
										self.assertEqual(
										    lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}				)
										self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]				)				)       ,							1.0				)

										snake_case_				:					Optional[int]   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health"]				)
										self.assertEqual(
										    lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}				)
										self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]				)				)       ,							1.0				)

										snake_case_				:					str   =       classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels="politics"       ,							hypothesis_template="This text is about {}"				)
										self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

										# https://github.com/huggingface/transformers/issues/13846
										snake_case_				:					Dict   =       classifier(["I am happy"]       ,							["positive", "negative"]				)
										self.assertEqual(
										    lowerCAmelCase__       ,							[
										        {"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}
										        for i in range(1				)
										    ]       ,							)
										snake_case_				:					Tuple   =       classifier(["I am happy", "I am sad"]       ,							["positive", "negative"]				)
										self.assertEqual(
										    lowerCAmelCase__       ,							[
										        {"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}
										        for i in range(2				)
										    ]       ,							)

										with self.assertRaises(lowerCAmelCase__				):
													classifier(""       ,							candidate_labels="politics"				)

										with self.assertRaises(lowerCAmelCase__				):
													classifier(lowerCAmelCase__       ,							candidate_labels="politics"				)

										with self.assertRaises(lowerCAmelCase__				):
													classifier("Who are you voting for in 2020?"       ,							candidate_labels=""				)

										with self.assertRaises(lowerCAmelCase__				):
													classifier("Who are you voting for in 2020?"       ,							candidate_labels=lowerCAmelCase__				)

										with self.assertRaises(lowerCAmelCase__				):
													classifier(
													    "Who are you voting for in 2020?"       ,							candidate_labels="politics"       ,							hypothesis_template="Not formatting template"       ,							)

										with self.assertRaises(lowerCAmelCase__				):
													classifier(
													    "Who are you voting for in 2020?"       ,							candidate_labels="politics"       ,							hypothesis_template=lowerCAmelCase__       ,							)

										self.run_entailment_id(lowerCAmelCase__				)

							def 							_A	(		self			:List[Any]       ,							lowerCAmelCase__			:Pipeline				)				->  Union[str, Any]:


										'''simple docstring'''




										snake_case_				:					int   =       zero_shot_classifier.model.config
										snake_case_				:					Optional[int]   =       config.labelaid
										snake_case_				:					Tuple   =       zero_shot_classifier.entailment_id

										snake_case_				:					Optional[Any]   =       {"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}
										self.assertEqual(zero_shot_classifier.entailment_id       ,							-1				)

										snake_case_				:					Tuple   =       {"entailment": 0, "neutral": 1, "contradiction": 2}
										self.assertEqual(zero_shot_classifier.entailment_id       ,							0				)

										snake_case_				:					str   =       {"ENTAIL": 0, "NON-ENTAIL": 1}
										self.assertEqual(zero_shot_classifier.entailment_id       ,							0				)

										snake_case_				:					str   =       {"ENTAIL": 2, "NEUTRAL": 1, "CONTR": 0}
										self.assertEqual(zero_shot_classifier.entailment_id       ,							2				)

										snake_case_				:					List[str]   =       original_labelaid
										self.assertEqual(lowerCAmelCase__       ,							zero_shot_classifier.entailment_id				)

							@require_torch
							def 							_A	(		self			:Tuple				)				->  Any:


										'''simple docstring'''




										snake_case_				:					List[Any]   =       pipeline(
										    "zero-shot-classification"       ,							model="sshleifer/tiny-distilbert-base-cased-distilled-squad"       ,							framework="pt"       ,							)
										# There was a regression in 4.10 for this
										# Adding a test so we don't make the mistake again.
										# https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499
										zero_shot_classifier(
										    "Who are you voting for in 2020?" * 100       ,							candidate_labels=["politics", "public health", "science"]				)

							@require_torch
							def 							_A	(		self			:Optional[Any]				)				->  Tuple:


										'''simple docstring'''




										snake_case_				:					Union[str, Any]   =       pipeline(
										    "zero-shot-classification"       ,							model="sshleifer/tiny-distilbert-base-cased-distilled-squad"       ,							framework="pt"       ,							)
										snake_case_				:					int   =       zero_shot_classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": "Who are you voting for in 2020?",
										        "labels": ["science", "public health", "politics"],
										        "scores": [0.3_3_3, 0.3_3_3, 0.3_3_3],
										    }       ,							)

							@require_tf
							def 							_A	(		self			:Union[str, Any]				)				->  Dict:


										'''simple docstring'''




										snake_case_				:					List[str]   =       pipeline(
										    "zero-shot-classification"       ,							model="sshleifer/tiny-distilbert-base-cased-distilled-squad"       ,							framework="tf"       ,							)
										snake_case_				:					Optional[int]   =       zero_shot_classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": "Who are you voting for in 2020?",
										        "labels": ["science", "public health", "politics"],
										        "scores": [0.3_3_3, 0.3_3_3, 0.3_3_3],
										    }       ,							)

							@slow
							@require_torch
							def 							_A	(		self			:Union[str, Any]				)				->  int:


										'''simple docstring'''




										snake_case_				:					int   =       pipeline("zero-shot-classification"       ,							model="roberta-large-mnli"       ,							framework="pt"				)
										snake_case_				:					str   =       zero_shot_classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": "Who are you voting for in 2020?",
										        "labels": ["politics", "public health", "science"],
										        "scores": [0.9_7_6, 0.0_1_5, 0.0_0_9],
										    }       ,							)
										snake_case_				:					Optional[int]   =       zero_shot_classifier(
										    "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
										    " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
										    " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
										    " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
										    " machine translation tasks show these models to be superior in quality while being more parallelizable"
										    " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
										    " English-to-German translation task, improving over the existing best results, including ensembles by"
										    " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
										    " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
										    " fraction of the training costs of the best models from the literature. We show that the Transformer"
										    " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
										    " large and limited training data."       ,							candidate_labels=["machine learning", "statistics", "translation", "vision"]       ,							multi_label=lowerCAmelCase__       ,							)
										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": (
										            "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
										            " networks in an encoder-decoder configuration. The best performing models also connect the"
										            " encoder and decoder through an attention mechanism. We propose a new simple network"
										            " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
										            " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
										            " superior in quality while being more parallelizable and requiring significantly less time to"
										            " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
										            " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
										            " English-to-French translation task, our model establishes a new single-model state-of-the-art"
										            " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
										            " costs of the best models from the literature. We show that the Transformer generalizes well to"
										            " other tasks by applying it successfully to English constituency parsing both with large and"
										            " limited training data."
										        ),
										        "labels": ["translation", "machine learning", "vision", "statistics"],
										        "scores": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
										    }       ,							)






							@slow
							@require_tf
							def 							_A	(		self			:List[str]				)				->  str:


										'''simple docstring'''




										snake_case_				:					int   =       pipeline("zero-shot-classification"       ,							model="roberta-large-mnli"       ,							framework="tf"				)
										snake_case_				:					Optional[Any]   =       zero_shot_classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": "Who are you voting for in 2020?",
										        "labels": ["politics", "public health", "science"],
										        "scores": [0.9_7_6, 0.0_1_5, 0.0_0_9],
										    }       ,							)
										snake_case_				:					Tuple   =       zero_shot_classifier(
										    "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
										    " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
										    " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
										    " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
										    " machine translation tasks show these models to be superior in quality while being more parallelizable"
										    " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
										    " English-to-German translation task, improving over the existing best results, including ensembles by"
										    " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
										    " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
										    " fraction of the training costs of the best models from the literature. We show that the Transformer"
										    " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
										    " large and limited training data."       ,							candidate_labels=["machine learning", "statistics", "translation", "vision"]       ,							multi_label=lowerCAmelCase__       ,							)
										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": (
										            "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
										            " networks in an encoder-decoder configuration. The best performing models also connect the"
										            " encoder and decoder through an attention mechanism. We propose a new simple network"
										            " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
										            " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
										            " superior in quality while being more parallelizable and requiring significantly less time to"
										            " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
										            " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
										            " English-to-French translation task, our model establishes a new single-model state-of-the-art"
										            " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
										            " costs of the best models from the literature. We show that the Transformer generalizes well to"
										            " other tasks by applying it successfully to English constituency parsing both with large and"
										            " limited training data."
										        ),
										        "labels": ["translation", "machine learning", "vision", "statistics"],
										        "scores": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
										    }       ,							)




