



'''simple docstring'''




import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a     :    int      						=					logging.get_logger(__name__)

a     :    str      						=					{
    """vocab_file""": """vocab.txt""",
    """merges_file""": """bpe.codes""",
}

a     :    Dict      						=					{
    """vocab_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt""",
    },
    """merges_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes""",
    },
}

a     :    Tuple      						=					{
    """vinai/phobert-base""": 2_5_6,
    """vinai/phobert-large""": 2_5_6,
}



def    __lowerCamelCase						(      _lowercase	)   ->    int:
							UpperCAmelCase  :	List[str]							  =	set()
							UpperCAmelCase  :	Union[str, Any]							  =	word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)	)
														UpperCAmelCase  :	Union[str, Any]							  =	char

							UpperCAmelCase  :	str							  =	set(_lowercase	)
							return pairs


class     UpperCamelCase_				(							__magic_name__    ):
	lowercase    =				VOCAB_FILES_NAMES
	lowercase    =				PRETRAINED_VOCAB_FILES_MAP
	lowercase    =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES





	def __init__(						self	,	A	,	A	,	A="<s>"	,	A="</s>"	,	A="</s>"	,	A="<s>"	,	A="<unk>"	,	A="<pad>"	,	A="<mask>"	,	**A	,	)    ->    List[str]:
								super().__init__(
								    bos_token=A	,	eos_token=A	,	unk_token=A	,	sep_token=A	,	cls_token=A	,	pad_token=A	,	mask_token=A	,	**A	,	)

								UpperCAmelCase  :	str							  =	vocab_file
								UpperCAmelCase  :	Any							  =	merges_file

								UpperCAmelCase  :	Optional[int]							  =	{}
								UpperCAmelCase  :	Dict							  =	0
								UpperCAmelCase  :	List[Any]							  =	1
								UpperCAmelCase  :	int							  =	2
								UpperCAmelCase  :	Tuple							  =	3

								self.add_from_file(A					)

								UpperCAmelCase  :	Optional[Any]							  =	{v: k for k, v in self.encoder.items()}

								with open(A	,	encoding="""utf-8"""					) as merges_handle:
															UpperCAmelCase  :	str							  =	merges_handle.read().split("""\n"""					)[:-1]
								UpperCAmelCase  :	List[Any]							  =	[tuple(merge.split()[:-1]					) for merge in merges]
								UpperCAmelCase  :	Any							  =	dict(zip(A	,	range(len(A					)					)					)					)
								UpperCAmelCase  :	Union[str, Any]							  =	{}





	def 				_lowercase(						self	,	A	,	A = None					)    ->    List[int]:

								if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								UpperCAmelCase  :	Optional[Any]							  =	[self.cls_token_id]
								UpperCAmelCase  :	Tuple							  =	[self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep





	def 				_lowercase(						self	,	A	,	A = None	,	A = False					)    ->    List[int]:

								if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=A	,	token_ids_a=A	,	already_has_special_tokens=A					)

								if token_ids_a is None:
															return [1] + ([0] * len(A					)) + [1]
								return [1] + ([0] * len(A					)) + [1, 1] + ([0] * len(A					)) + [1]





	def 				_lowercase(						self	,	A	,	A = None					)    ->    List[int]:
								UpperCAmelCase  :	Tuple							  =	[self.sep_token_id]
								UpperCAmelCase  :	str							  =	[self.cls_token_id]

								if token_ids_a is None:
															return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]





	@property
	def 				_lowercase(						self					)    ->    Tuple:
								return len(self.encoder					)





	def 				_lowercase(						self					)    ->    Tuple:
								return dict(self.encoder	,	**self.added_tokens_encoder					)





	def 				_lowercase(						self	,	A					)    ->    Any:
								if token in self.cache:
															return self.cache[token]
								UpperCAmelCase  :	Optional[int]							  =	tuple(A					)
								UpperCAmelCase  :	List[str]							  =	tuple(list(word[:-1]					) + [word[-1] + """</w>"""]					)
								UpperCAmelCase  :	Union[str, Any]							  =	get_pairs(A					)

								if not pairs:
															return token

								while True:
															UpperCAmelCase  :	Dict							  =	min(A	,	key=lambda A					: self.bpe_ranks.get(A	,	float("""inf"""					)					)					)
															if bigram not in self.bpe_ranks:
																						break
															UpperCAmelCase  ,  UpperCAmelCase  :	int							  =	bigram
															UpperCAmelCase  :	List[Any]							  =	[]
															UpperCAmelCase  :	Union[str, Any]							  =	0
															while i < len(A					):
																						try:
																													UpperCAmelCase  :	Any							  =	word.index(A	,	A					)
																						except ValueError:
																													new_word.extend(word[i:]					)
																													break
																						else:
																													new_word.extend(word[i:j]					)
																													UpperCAmelCase  :	Optional[int]							  =	j

																						if word[i] == first and i < len(A					) - 1 and word[i + 1] == second:
																													new_word.append(first + second					)
																													i += 2
																						else:
																													new_word.append(word[i]					)
																													i += 1
															UpperCAmelCase  :	Optional[Any]							  =	tuple(A					)
															UpperCAmelCase  :	List[Any]							  =	new_word
															if len(A					) == 1:
																						break
															else:
																						UpperCAmelCase  :	Optional[int]							  =	get_pairs(A					)
								UpperCAmelCase  :	List[str]							  =	"""@@ """.join(A					)
								UpperCAmelCase  :	List[str]							  =	word[:-4]
								UpperCAmelCase  :	int							  =	word
								return word





	def 				_lowercase(						self	,	A					)    ->    List[Any]:
								UpperCAmelCase  :	Dict							  =	[]

								UpperCAmelCase  :	Optional[Any]							  =	re.findall(r"""\S+\n?"""	,	A					)

								for token in words:
															split_tokens.extend(list(self.bpe(A					).split(""" """					)					)					)
								return split_tokens





	def 				_lowercase(						self	,	A					)    ->    Dict:
								return self.encoder.get(A	,	self.encoder.get(self.unk_token					)					)





	def 				_lowercase(						self	,	A					)    ->    Dict:
								return self.decoder.get(A	,	self.unk_token					)





	def 				_lowercase(						self	,	A					)    ->    Optional[Any]:
								UpperCAmelCase  :	List[Any]							  =	""" """.join(A					).replace("""@@ """	,	""""""					).strip()
								return out_string





	def 				_lowercase(						self	,	A	,	A = None					)    ->    Tuple[str]:
								if not os.path.isdir(A					):
															logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
															return
								UpperCAmelCase  :	List[Any]							  =	os.path.join(
								    A	,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
								UpperCAmelCase  :	Optional[Any]							  =	os.path.join(
								    A	,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]					)

								if os.path.abspath(self.vocab_file					) != os.path.abspath(A					):
															copyfile(self.vocab_file	,	A					)

								if os.path.abspath(self.merges_file					) != os.path.abspath(A					):
															copyfile(self.merges_file	,	A					)

								return out_vocab_file, out_merge_file






	def 				_lowercase(						self	,	A					)    ->    List[str]:
								if isinstance(A	,	A					):
															try:
																						with open(A	,	"""r"""	,	encoding="""utf-8"""					) as fd:
																													self.add_from_file(A					)
															except FileNotFoundError as fnfe:
																						raise fnfe
															except UnicodeError:
																						raise Exception(f'''Incorrect encoding detected in {f}, please rebuild the dataset'''					)
															return

								UpperCAmelCase  :	int							  =	f.readlines()
								for lineTmp in lines:
															UpperCAmelCase  :	int							  =	lineTmp.strip()
															UpperCAmelCase  :	Optional[Any]							  =	line.rfind(""" """					)
															if idx == -1:
																						raise ValueError("""Incorrect dictionary format, expected '<token> <cnt>'"""					)
															UpperCAmelCase  :	int							  =	line[:idx]
															UpperCAmelCase  :	Union[str, Any]							  =	len(self.encoder					)






'''simple docstring'''




from sklearn.metrics import mean_squared_error

import datasets


a     :    Union[str, Any]      						=					"""\
@article{scikit-learn,
 title={Scikit-learn: Machine Learning in {P}ython},
 author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
 journal={Journal of Machine Learning Research},
 volume={12},
 pages={2825--2830},
 year={2011}
}
"""

a     :    int      						=					"""\
Mean Squared Error(MSE) is the average of the square of difference between the predicted
and actual values.
"""


a     :    int      						=					"""
Args:
    predictions: array-like of shape (n_samples,) or (n_samples, n_outputs)
        Estimated target values.
    references: array-like of shape (n_samples,) or (n_samples, n_outputs)
        Ground truth (correct) target values.
    sample_weight: array-like of shape (n_samples,), default=None
        Sample weights.
    multioutput: {\"raw_values\", \"uniform_average\"} or array-like of shape (n_outputs,), default=\"uniform_average\"
        Defines aggregating of multiple output values. Array-like value defines weights used to average errors.

        \"raw_values\" : Returns a full set of errors in case of multioutput input.

        \"uniform_average\" : Errors of all outputs are averaged with uniform weight.

    squared : bool, default=True
        If True returns MSE value, if False returns RMSE (Root Mean Squared Error) value.

Returns:
    mse : mean squared error.
Examples:

    >>> mse_metric = datasets.load_metric(\"mse\")
    >>> predictions = [2.5, 0.0, 2, 8]
    >>> references = [3, -0.5, 2, 7]
    >>> results = mse_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'mse': 0.375}
    >>> rmse_result = mse_metric.compute(predictions=predictions, references=references, squared=False)
    >>> print(rmse_result)
    {'mse': 0.6123724356957945}

    If you're using multi-dimensional lists, then set the config as follows :

    >>> mse_metric = datasets.load_metric(\"mse\", \"multilist\")
    >>> predictions = [[0.5, 1], [-1, 1], [7, -6]]
    >>> references = [[0, 2], [-1, 2], [8, -5]]
    >>> results = mse_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'mse': 0.7083333333333334}
    >>> results = mse_metric.compute(predictions=predictions, references=references, multioutput='raw_values')
    >>> print(results) # doctest: +NORMALIZE_WHITESPACE
    {'mse': array([0.41666667, 1. ])}
"""

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,  _KWARGS_DESCRIPTION    )
class     UpperCamelCase_				(							datasets.Metric    ):





	def 				_lowercase(						self					)    ->    List[str]:
								return datasets.MetricInfo(
								    description=_DESCRIPTION	,	citation=_CITATION	,	inputs_description=_KWARGS_DESCRIPTION	,	features=datasets.Features(self._get_feature_types()					)	,	reference_urls=[
								        """https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html"""
								    ]	,	)





	def 				_lowercase(						self					)    ->    List[Any]:
								if self.config_name == "multilist":
															return {
															    "predictions": datasets.Sequence(datasets.Value("""float"""					)					),
															    "references": datasets.Sequence(datasets.Value("""float"""					)					),
															}
								else:
															return {
															    "predictions": datasets.Value("""float"""					),
															    "references": datasets.Value("""float"""					),
															}






	def 				_lowercase(						self	,	A	,	A	,	A=None	,	A="uniform_average"	,	A=True					)    ->    List[Any]:
								UpperCAmelCase  :	List[Any]							  =	mean_squared_error(
								    A	,	A	,	sample_weight=A	,	multioutput=A	,	squared=A					)

								return {"mse": mse}

