

"""simple docstring"""




import comet  # From: unbabel-comet
import torch

import datasets


A_							:			List[Any]											= datasets.logging.get_logger(__name__)

A_							:			str											= "\\n@inproceedings{rei-EtAl:2020:WMT,\n   author    = {Rei, Ricardo  and  Stewart, Craig  and  Farinha, Ana C  and  Lavie, Alon},\n   title     = {Unbabel's Participation in the WMT20 Metrics Shared Task},\n   booktitle      = {Proceedings of the Fifth Conference on Machine Translation},\n   month          = {November},\n   year           = {2020},\n   address        = {Online},\n   publisher      = {Association for Computational Linguistics},\n   pages     = {909--918},\n}\n@inproceedings{rei-etal-2020-comet,\n   title = \"{COMET}: A Neural Framework for {MT} Evaluation\",\n   author = \"Rei, Ricardo  and\n      Stewart, Craig  and\n      Farinha, Ana C  and\n      Lavie, Alon\",\n   booktitle = \"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)\",\n   month = nov,\n   year = \"2020\",\n   address = \"Online\",\n   publisher = \"Association for Computational Linguistics\",\n   url = \"https://www.aclweb.org/anthology/2020.emnlp-main.213\",\n   pages = \"2685--2702\",\n}\n"

A_							:			int											= "\\nCrosslingual Optimized Metric for Evaluation of Translation (COMET) is an open-source framework used to train Machine Translation metrics that achieve high levels of correlation with different types of human judgments (HTER, DA's or MQM).\nWith the release of the framework the authors also released fully trained models that were used to compete in the WMT20 Metrics Shared Task achieving SOTA in that years competition.\n\nSee the [README.md] file at https://unbabel.github.io/COMET/html/models.html for more information.\n"

A_							:			Union[str, Any]											= "\nCOMET score.\n\nArgs:\n\n`sources` (list of str): Source sentences\n`predictions` (list of str): candidate translations\n`references` (list of str): reference translations\n`cuda` (bool): If set to True, runs COMET using GPU\n`show_progress` (bool): Shows progress\n`model`: COMET model to be used. Will default to `wmt-large-da-estimator-1719` if None.\n\nReturns:\n    `samples`: List of dictionaries with `src`, `mt`, `ref` and `score`.\n    `scores`: List of scores.\n\nExamples:\n\n    >>> comet_metric = datasets.load_metric('comet')\n    >>> # comet_metric = load_metric('comet', 'wmt20-comet-da')  # you can also choose which model to use\n    >>> source = [\"Dem Feuer konnte Einhalt geboten werden\", \"Schulen und Kindergärten wurden eröffnet.\"]\n    >>> hypothesis = [\"The fire could be stopped\", \"Schools and kindergartens were open\"]\n    >>> reference = [\"They were able to control the fire.\", \"Schools and kindergartens opened\"]\n    >>> results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source)\n    >>> print([round(v, 2) for v in results[\"scores\"]])\n    [0.19, 0.92]\n"





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,_KWARGS_DESCRIPTION			)
class    lowerCamelCase						(datasets.Metric			):

			def 					SCREAMING_SNAKE_CASE				(					self				:						Tuple       )   ->    int:
						return datasets.MetricInfo(
						    description=_DESCRIPTION    ,				citation=_CITATION    ,				homepage="""https://unbabel.github.io/COMET/html/index.html"""    ,				inputs_description=_KWARGS_DESCRIPTION    ,				features=datasets.Features(
						        {
						            """sources""": datasets.Value("""string"""    ,				id="""sequence"""       ),
						            """predictions""": datasets.Value("""string"""    ,				id="""sequence"""       ),
						            """references""": datasets.Value("""string"""    ,				id="""sequence"""       ),
						        }       )    ,				codebase_urls=["""https://github.com/Unbabel/COMET"""]    ,				reference_urls=[
						        """https://github.com/Unbabel/COMET""",
						        """https://www.aclweb.org/anthology/2020.emnlp-main.213/""",
						        """http://www.statmt.org/wmt20/pdf/2020.wmt-1.101.pdf6""",
						    ]    ,				)

			def 					SCREAMING_SNAKE_CASE				(					self				:						Tuple    ,				__UpperCAmelCase				:						str       )   ->    str:
						if self.config_name == "default":
									SCREAMING_SNAKE_CASE__								=      comet.load_from_checkpoint(comet.download_model("""wmt20-comet-da"""       )       )
						else:
									SCREAMING_SNAKE_CASE__								=      comet.load_from_checkpoint(comet.download_model(self.config_name       )       )





			def 					SCREAMING_SNAKE_CASE				(					self				:						str    ,				__UpperCAmelCase				:						Tuple    ,				__UpperCAmelCase				:						Tuple    ,				__UpperCAmelCase				:						int    ,				__UpperCAmelCase				:						Any=None    ,				__UpperCAmelCase				:						List[str]=False       )   ->    List[Any]:
						if gpus is None:
									SCREAMING_SNAKE_CASE__								=      1 if torch.cuda.is_available() else 0
						SCREAMING_SNAKE_CASE__								=      {"""src""": sources, """mt""": predictions, """ref""": references}
						SCREAMING_SNAKE_CASE__								=      [dict(zip(__UpperCAmelCase    ,				__UpperCAmelCase       )       ) for t in zip(*data.values()       )]
						SCREAMING_SNAKE_CASE__							,		SCREAMING_SNAKE_CASE__								=      self.scorer.predict(__UpperCAmelCase    ,				gpus=__UpperCAmelCase    ,				progress_bar=__UpperCAmelCase       )
						return {"mean_score": mean_score, "scores": scores}





"""simple docstring"""




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A_							:			Optional[int]											= logging.get_logger(__name__)

A_							:			Union[str, Any]											= {"vocab_file": "sentencepiece.bpe.model"}

A_							:			str											= {
    "vocab_file": {
        "camembert-base": "https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model",
    }
}

A_							:			Tuple											= {
    "camembert-base": 512,
}

A_							:			int											= "▁"





class    lowerCamelCase						(A__			):
			lowerCamelCase__   :		int							=							VOCAB_FILES_NAMES
			lowerCamelCase__   :		List[Any]							=							PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__   :		List[str]							=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__   :		Optional[int]							=							['input_ids', 'attention_mask']

			def __init__(					self				:						Tuple    ,				__UpperCAmelCase				:						Optional[Any]    ,				__UpperCAmelCase				:						int="<s>"    ,				__UpperCAmelCase				:						Union[str, Any]="</s>"    ,				__UpperCAmelCase				:						List[str]="</s>"    ,				__UpperCAmelCase				:						List[Any]="<s>"    ,				__UpperCAmelCase				:						int="<unk>"    ,				__UpperCAmelCase				:						Dict="<pad>"    ,				__UpperCAmelCase				:						int="<mask>"    ,				__UpperCAmelCase				:						List[Any]=["<s>NOTUSED", "</s>NOTUSED"]    ,				__UpperCAmelCase				:						Optional[Dict[str, Any]] = None    ,				**__UpperCAmelCase				:						Optional[Any]    ,				)   ->    None:
						# Mask token behave like a normal word, i.e. include the space before it
						SCREAMING_SNAKE_CASE__								=      AddedToken(__UpperCAmelCase    ,				lstrip=__UpperCAmelCase    ,				rstrip=__UpperCAmelCase       ) if isinstance(__UpperCAmelCase    ,				__UpperCAmelCase       ) else mask_token

						SCREAMING_SNAKE_CASE__								=      {} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    bos_token=__UpperCAmelCase    ,				eos_token=__UpperCAmelCase    ,				unk_token=__UpperCAmelCase    ,				sep_token=__UpperCAmelCase    ,				cls_token=__UpperCAmelCase    ,				pad_token=__UpperCAmelCase    ,				mask_token=__UpperCAmelCase    ,				additional_special_tokens=__UpperCAmelCase    ,				sp_model_kwargs=self.sp_model_kwargs    ,				**__UpperCAmelCase    ,				)
						SCREAMING_SNAKE_CASE__								=      spm.SentencePieceProcessor(**self.sp_model_kwargs       )
						self.sp_model.Load(str(__UpperCAmelCase       )       )
						SCREAMING_SNAKE_CASE__								=      vocab_file
						# HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
						# sentencepiece vocabulary (this is the case for <s> and </s>
						SCREAMING_SNAKE_CASE__								=      {"""<s>NOTUSED""": 0, """<pad>""": 1, """</s>NOTUSED""": 2, """<unk>""": 3}
						SCREAMING_SNAKE_CASE__								=      len(self.fairseq_tokens_to_ids       )
						SCREAMING_SNAKE_CASE__								=      len(self.sp_model       ) + len(self.fairseq_tokens_to_ids       )
						SCREAMING_SNAKE_CASE__								=      {v: k for k, v in self.fairseq_tokens_to_ids.items()}

			def 					SCREAMING_SNAKE_CASE				(					self				:						Optional[int]    ,				__UpperCAmelCase				:						List[int]    ,				__UpperCAmelCase				:						Optional[List[int]] = None       )   ->    List[int]:

						if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						SCREAMING_SNAKE_CASE__								=      [self.cls_token_id]
						SCREAMING_SNAKE_CASE__								=      [self.sep_token_id]
						return cls + token_ids_a + sep + sep + token_ids_a + sep

			def 					SCREAMING_SNAKE_CASE				(					self				:						Optional[int]    ,				__UpperCAmelCase				:						List[int]    ,				__UpperCAmelCase				:						Optional[List[int]] = None    ,				__UpperCAmelCase				:						bool = False       )   ->    List[int]:
						if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=__UpperCAmelCase    ,				token_ids_a=__UpperCAmelCase    ,				already_has_special_tokens=__UpperCAmelCase       )

						if token_ids_a is None:
									return [1] + ([0] * len(__UpperCAmelCase       )) + [1]
						return [1] + ([0] * len(__UpperCAmelCase       )) + [1, 1] + ([0] * len(__UpperCAmelCase       )) + [1]

			def 					SCREAMING_SNAKE_CASE				(					self				:						Tuple    ,				__UpperCAmelCase				:						List[int]    ,				__UpperCAmelCase				:						Optional[List[int]] = None       )   ->    List[int]:
						SCREAMING_SNAKE_CASE__								=      [self.sep_token_id]
						SCREAMING_SNAKE_CASE__								=      [self.cls_token_id]

						if token_ids_a is None:
									return len(cls + token_ids_a + sep       ) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]

			@property
			def 					SCREAMING_SNAKE_CASE				(					self				:						Any       )   ->    Any:
						return len(self.fairseq_tokens_to_ids       ) + len(self.sp_model       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						int       )   ->    Optional[Any]:
						SCREAMING_SNAKE_CASE__								=      {self.convert_ids_to_tokens(__UpperCAmelCase       ): i for i in range(self.vocab_size       )}
						vocab.update(self.added_tokens_encoder       )
						return vocab

			def 					SCREAMING_SNAKE_CASE				(					self				:						Optional[int]    ,				__UpperCAmelCase				:						str       )   ->    List[str]:
						return self.sp_model.encode(__UpperCAmelCase    ,				out_type=__UpperCAmelCase       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						Tuple    ,				__UpperCAmelCase				:						Tuple       )   ->    Optional[Any]:
						if token in self.fairseq_tokens_to_ids:
									return self.fairseq_tokens_to_ids[token]
						elif self.sp_model.PieceToId(__UpperCAmelCase       ) == 0:
									# Convert sentence piece unk token to fairseq unk token index
									return self.unk_token_id
						return self.fairseq_offset + self.sp_model.PieceToId(__UpperCAmelCase       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						Optional[Any]    ,				__UpperCAmelCase				:						int       )   ->    Dict:
						if index in self.fairseq_ids_to_tokens:
									return self.fairseq_ids_to_tokens[index]
						return self.sp_model.IdToPiece(index - self.fairseq_offset       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						Optional[int]    ,				__UpperCAmelCase				:						List[str]       )   ->    Union[str, Any]:
						SCREAMING_SNAKE_CASE__								=      []
						SCREAMING_SNAKE_CASE__								=      """"""
						SCREAMING_SNAKE_CASE__								=      False
						for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
												if not prev_is_special:
															out_string += " "
												out_string += self.sp_model.decode(__UpperCAmelCase       ) + token
												SCREAMING_SNAKE_CASE__								=      True
												SCREAMING_SNAKE_CASE__								=      []
									else:
												current_sub_tokens.append(__UpperCAmelCase       )
												SCREAMING_SNAKE_CASE__								=      False
						out_string += self.sp_model.decode(__UpperCAmelCase       )
						return out_string.strip()

			def __getstate__(					self				:						Optional[Any]       )   ->    Any:
						SCREAMING_SNAKE_CASE__								=      self.__dict__.copy()
						SCREAMING_SNAKE_CASE__								=      None
						return state

			def __setstate__(					self				:						int    ,				__UpperCAmelCase				:						Optional[int]       )   ->    str:
						SCREAMING_SNAKE_CASE__								=      d

						# for backward compatibility
						if not hasattr(self    ,				"""sp_model_kwargs"""       ):
									SCREAMING_SNAKE_CASE__								=      {}

						SCREAMING_SNAKE_CASE__								=      spm.SentencePieceProcessor(**self.sp_model_kwargs       )
						self.sp_model.Load(self.vocab_file       )





			def 					SCREAMING_SNAKE_CASE				(					self				:						Optional[Any]    ,				__UpperCAmelCase				:						str    ,				__UpperCAmelCase				:						Optional[str] = None       )   ->    Tuple[str]:
						if not os.path.isdir(__UpperCAmelCase       ):
									logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
									return
						SCREAMING_SNAKE_CASE__								=      os.path.join(
						    __UpperCAmelCase    ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )

						if os.path.abspath(self.vocab_file       ) != os.path.abspath(__UpperCAmelCase       ) and os.path.isfile(self.vocab_file       ):
									copyfile(self.vocab_file    ,				__UpperCAmelCase       )
						elif not os.path.isfile(self.vocab_file       ):
									with open(__UpperCAmelCase    ,				"""wb"""       ) as fi:
												SCREAMING_SNAKE_CASE__								=      self.sp_model.serialized_model_proto()
												fi.write(__UpperCAmelCase       )

						return (out_vocab_file,)


