






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


__lowerCAmelCase       		=      logging.get_logger(__name__)


__lowerCAmelCase       		=      {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

# See all BART models at https://huggingface.co/models?filter=bart
__lowerCAmelCase       		=      {
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json',
    },
}

__lowerCAmelCase       		=      {
    'facebook/bart-base': 1_024,
    'facebook/bart-large': 1_024,
    'facebook/bart-large-mnli': 1_024,
    'facebook/bart-large-cnn': 1_024,
    'facebook/bart-large-xsum': 1_024,
    'yjernite/bart_eli5': 1_024,
}





class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				VOCAB_FILES_NAMES
			snake_case__  =				PRETRAINED_VOCAB_FILES_MAP
			snake_case__  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case__  =				["input_ids", "attention_mask"]
			snake_case__  =				BartTokenizer






			def __init__( self      :			Dict  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=None  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=None  ,						__SCREAMING_SNAKE_CASE      :			Any=None  ,						__SCREAMING_SNAKE_CASE      :			Tuple="replace"  ,						__SCREAMING_SNAKE_CASE      :			Dict="<s>"  ,						__SCREAMING_SNAKE_CASE      :			int="</s>"  ,						__SCREAMING_SNAKE_CASE      :			Tuple="</s>"  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]="<s>"  ,						__SCREAMING_SNAKE_CASE      :			Any="<unk>"  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]="<pad>"  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]="<mask>"  ,						__SCREAMING_SNAKE_CASE      :			Tuple=False  ,						__SCREAMING_SNAKE_CASE      :			int=True  ,						**__SCREAMING_SNAKE_CASE      :			Tuple  ,						)			->			Tuple:
							super().__init__(
							    __SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE  ,						tokenizer_file=__SCREAMING_SNAKE_CASE  ,						errors=__SCREAMING_SNAKE_CASE  ,						bos_token=__SCREAMING_SNAKE_CASE  ,						eos_token=__SCREAMING_SNAKE_CASE  ,						sep_token=__SCREAMING_SNAKE_CASE  ,						cls_token=__SCREAMING_SNAKE_CASE  ,						unk_token=__SCREAMING_SNAKE_CASE  ,						pad_token=__SCREAMING_SNAKE_CASE  ,						mask_token=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE  ,						)

							a_		:  List[str]											=      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
							if pre_tok_state.get('''add_prefix_space'''  ,						__SCREAMING_SNAKE_CASE					) != add_prefix_space:
											a_		:  Any											=      getattr(__SCREAMING_SNAKE_CASE  ,						pre_tok_state.pop('''type'''					)					)
											a_		:  int											=      add_prefix_space
											a_		:  Union[str, Any]											=      pre_tok_class(**__SCREAMING_SNAKE_CASE					)

							a_		:  List[str]											=      add_prefix_space

							# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
							a_		:  int											=      '''post_processor'''
							a_		:  Optional[int]											=      getattr(self.backend_tokenizer  ,						__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)
							if tokenizer_component_instance:
											a_		:  Optional[int]											=      json.loads(tokenizer_component_instance.__getstate__()					)

											# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
											if "sep" in state:
															a_		:  Optional[int]											=      tuple(state['''sep''']					)
											if "cls" in state:
															a_		:  List[str]											=      tuple(state['''cls''']					)

											a_		:  List[str]											=      False

											if state.get('''add_prefix_space'''  ,						__SCREAMING_SNAKE_CASE					) != add_prefix_space:
															a_		:  Optional[Any]											=      add_prefix_space
															a_		:  str											=      True

											if state.get('''trim_offsets'''  ,						__SCREAMING_SNAKE_CASE					) != trim_offsets:
															a_		:  Dict											=      trim_offsets
															a_		:  Any											=      True

											if changes_to_apply:
															a_		:  str											=      getattr(__SCREAMING_SNAKE_CASE  ,						state.pop('''type'''					)					)
															a_		:  List[Any]											=      component_class(**__SCREAMING_SNAKE_CASE					)
															setattr(self.backend_tokenizer  ,						__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			str:
							if self._mask_token is None:
											if self.verbose:
															logger.error('''Using mask_token, but it is not set yet.'''					)
											return None
							return str(self._mask_token					)






			@mask_token.setter
			def 			SCREAMING_SNAKE_CASE      ( self      :			Any  ,						__SCREAMING_SNAKE_CASE      :			Any					)			->			Dict:
							a_		:  int											=      AddedToken(__SCREAMING_SNAKE_CASE  ,						lstrip=__SCREAMING_SNAKE_CASE  ,						rstrip=__SCREAMING_SNAKE_CASE					) if isinstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					) else value
							a_		:  Any											=      value






			def 			SCREAMING_SNAKE_CASE      ( self      :			Any  ,						*__SCREAMING_SNAKE_CASE      :			Tuple  ,						**__SCREAMING_SNAKE_CASE      :			Optional[Any]					)			->			BatchEncoding:
							a_		:  Tuple											=      kwargs.get('''is_split_into_words'''  ,						__SCREAMING_SNAKE_CASE					)

							if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    '''to use it with pretokenized inputs.'''					)

							return super()._batch_encode_plus(*__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]  ,						*__SCREAMING_SNAKE_CASE      :			List[str]  ,						**__SCREAMING_SNAKE_CASE      :			Dict					)			->			BatchEncoding:
							a_		:  int											=      kwargs.get('''is_split_into_words'''  ,						__SCREAMING_SNAKE_CASE					)

							if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    '''to use it with pretokenized inputs.'''					)

							return super()._encode_plus(*__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[int]  ,						__SCREAMING_SNAKE_CASE      :			str  ,						__SCREAMING_SNAKE_CASE      :			Optional[str] = None					)			->			Tuple[str]:
							a_		:  List[Any]											=      self._tokenizer.model.save(__SCREAMING_SNAKE_CASE  ,						name=__SCREAMING_SNAKE_CASE					)
							return tuple(__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict  ,						__SCREAMING_SNAKE_CASE      :			str  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=None					)			->			Dict:
							a_		:  int											=      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
											return output

							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]  ,						__SCREAMING_SNAKE_CASE      :			List[int]  ,						__SCREAMING_SNAKE_CASE      :			Optional[List[int]] = None					)			->			List[int]:
							a_		:  Optional[Any]											=      [self.sep_token_id]
							a_		:  str											=      [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep					) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]












'''simple docstring'''







import re
import string

import numpy as np

import datasets


__lowerCAmelCase       		=      '\nReturns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.\n'

__lowerCAmelCase       		=      '\nArgs:\n    predictions: List of predicted texts.\n    references: List of reference texts.\n    regexes_to_ignore: List, defaults to None. Regex expressions of characters to\n        ignore when calculating the exact matches. Note: these regexes are removed\n        from the input data before the changes based on the options below (e.g. ignore_case,\n        ignore_punctuation, ignore_numbers) are applied.\n    ignore_case: Boolean, defaults to False. If true, turns everything\n        to lowercase so that capitalization differences are ignored.\n    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\n    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\nReturns:\n    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.\nExamples:\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    25.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    50.0\n\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    75.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)\n    >>> print(round(results["exact_match"], 1))\n    100.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It\'s like comparing oranges and apples."]\n    >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It\'s like comparing apples and oranges."]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    33.3\n\n'

__lowerCAmelCase       		=      '\n'





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,   _KWARGS_DESCRIPTION       )
class 	SCREAMING_SNAKE_CASE					(					datasets.Metric       ):






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			int:
							return datasets.MetricInfo(
							    description=_DESCRIPTION  ,						citation=_CITATION  ,						inputs_description=_KWARGS_DESCRIPTION  ,						features=datasets.Features(
							        {
							            '''predictions''': datasets.Value('''string'''  ,						id='''sequence'''					),
							            '''references''': datasets.Value('''string'''  ,						id='''sequence'''					),
							        }					)  ,						reference_urls=[]  ,						)







			def 			SCREAMING_SNAKE_CASE      ( self      :			str  ,						__SCREAMING_SNAKE_CASE      :			List[Any]  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]  ,						__SCREAMING_SNAKE_CASE      :			Dict=None  ,						__SCREAMING_SNAKE_CASE      :			int=False  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]=False  ,						__SCREAMING_SNAKE_CASE      :			Dict=False  ,						)			->			str:
							if regexes_to_ignore is not None:
											for s in regexes_to_ignore:
															a_		:  Optional[Any]											=      np.array([re.sub(__SCREAMING_SNAKE_CASE  ,						''''''  ,						__SCREAMING_SNAKE_CASE					) for x in predictions]					)
															a_		:  int											=      np.array([re.sub(__SCREAMING_SNAKE_CASE  ,						''''''  ,						__SCREAMING_SNAKE_CASE					) for x in references]					)
							else:
											a_		:  List[str]											=      np.asarray(__SCREAMING_SNAKE_CASE					)
											a_		:  Any											=      np.asarray(__SCREAMING_SNAKE_CASE					)

							if ignore_case:
											a_		:  List[str]											=      np.char.lower(__SCREAMING_SNAKE_CASE					)
											a_		:  List[Any]											=      np.char.lower(__SCREAMING_SNAKE_CASE					)

							if ignore_punctuation:
											a_		:  Any											=      string.punctuation.maketrans(''''''  ,						''''''  ,						string.punctuation					)
											a_		:  Union[str, Any]											=      np.char.translate(__SCREAMING_SNAKE_CASE  ,						table=__SCREAMING_SNAKE_CASE					)
											a_		:  int											=      np.char.translate(__SCREAMING_SNAKE_CASE  ,						table=__SCREAMING_SNAKE_CASE					)

							if ignore_numbers:
											a_		:  int											=      string.digits.maketrans(''''''  ,						''''''  ,						string.digits					)
											a_		:  Optional[int]											=      np.char.translate(__SCREAMING_SNAKE_CASE  ,						table=__SCREAMING_SNAKE_CASE					)
											a_		:  Dict											=      np.char.translate(__SCREAMING_SNAKE_CASE  ,						table=__SCREAMING_SNAKE_CASE					)

							a_		:  Optional[Any]											=      predictions == references

							return {"exact_match": np.mean(__SCREAMING_SNAKE_CASE					) * 100}




