




from jiwer import compute_measures

import datasets


lowerCamelCase       :	List[Any]									=     "\\n@inproceedings{inproceedings,\n    author = {Morris, Andrew and Maier, Viktoria and Green, Phil},\n    year = {2004},\n    month = {01},\n    pages = {},\n    title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.}\n}\n"

lowerCamelCase       :	Union[str, Any]									=     "\\nWord error rate (WER) is a common metric of the performance of an automatic speech recognition system.\n\nThe general difficulty of measuring performance lies in the fact that the recognized word sequence can have a different length from the reference word sequence (supposedly the correct one). The WER is derived from the Levenshtein distance, working at the word level instead of the phoneme level. The WER is a valuable tool for comparing different systems as well as for evaluating improvements within one system. This kind of measurement, however, provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort.\n\nThis problem is solved by first aligning the recognized word sequence with the reference (spoken) word sequence using dynamic string alignment. Examination of this issue is seen through a theory called the power law that states the correlation between perplexity and word error rate.\n\nWord error rate can then be computed as:\n\nWER = (S + D + I) / N = (S + D + I) / (S + D + C)\n\nwhere\n\nS is the number of substitutions,\nD is the number of deletions,\nI is the number of insertions,\nC is the number of correct words,\nN is the number of words in the reference (N=S+D+C).\n\nThis value indicates the average number of errors per reference word. The lower the value, the better the\nperformance of the ASR system with a WER of 0 being a perfect score.\n"

lowerCamelCase       :	Union[str, Any]									=     "\nCompute WER score of transcribed segments against references.\n\nArgs:\n    references: List of references for each speech input.\n    predictions: List of transcriptions to score.\n    concatenate_texts (bool, default=False): Whether to concatenate all input texts or compute WER iteratively.\n\nReturns:\n    (float): the word error rate\n\nExamples:\n\n    >>> predictions = [\"this is the prediction\", \"there is an other sample\"]\n    >>> references = [\"this is the reference\", \"there is another one\"]\n    >>> wer = datasets.load_metric(\"wer\")\n    >>> wer_score = wer.compute(predictions=predictions, references=references)\n    >>> print(wer_score)\n    0.5\n"







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,   _KWARGS_DESCRIPTION     )
class 						A(		datasets.Metric     ):



 '''simple docstring'''


 def        a__				(       self      :						Optional[int]						)     ->			List[Any]:

       """simple docstring"""






       return datasets.MetricInfo(
           description=_DESCRIPTION		,							citation=_CITATION		,							inputs_description=_KWARGS_DESCRIPTION		,							features=datasets.Features(
               {
                   'predictions': datasets.Value('string'		,							id='sequence'						),
                   'references': datasets.Value('string'		,							id='sequence'						),
               }						)		,							codebase_urls=['https://github.com/jitsi/jiwer/']		,							reference_urls=[
               'https://en.wikipedia.org/wiki/Word_error_rate',
           ]		,							)





 def        a__				(       self      :						int		,							A_      :						List[str]=None		,							A_      :						Union[str, Any]=None		,							A_      :						Dict=False						)     ->			Any:

       """simple docstring"""






       if concatenate_texts:
             return compute_measures(A_		,							A_						)["wer"]
       else:
             lowerCamelCase_	  =					0
             lowerCamelCase_	  =					0
             for prediction, reference in zip(A_		,							A_						):
                   lowerCamelCase_	  =					compute_measures(A_		,							A_						)
                   incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
                   total += measures["substitutions"] + measures["deletions"] + measures["hits"]
             return incorrect / total









import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation

lowerCamelCase       :	str									=     logging.get_logger(__name__)


lowerCamelCase       :	Optional[Any]									=     {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

lowerCamelCase       :	int									=     {
    "vocab_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"},
    "merges_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"},
    "tokenizer_config_file": {
        "facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"
    },
}

lowerCamelCase       :	Tuple									=     {"facebook/blenderbot-3B": 128}

@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def     _SCREAMING_SNAKE_CASE    (							):


      '''simple docstring'''



      lowerCamelCase_	  =					(
          list(range(ord('!'  )			,							ord('~'  ) + 1  )  ) + list(range(ord('¡'  )			,							ord('¬'  ) + 1  )  ) + list(range(ord('®'  )			,							ord('ÿ'  ) + 1  )  )
      )
      lowerCamelCase_	  =					bs[:]
      lowerCamelCase_	  =					0
      for b in range(2**8  ):
            if b not in bs:
                  bs.append(lowercase  )
                  cs.append(2**8 + n  )
                  n += 1
      lowerCamelCase_	  =					[chr(lowercase  ) for n in cs]
      return dict(zip(lowercase			,							lowercase  )  )

def     _SCREAMING_SNAKE_CASE    (							lowercase			: int  ):


      '''simple docstring'''



      lowerCamelCase_	  =					set()
      lowerCamelCase_	  =					word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)  )
            lowerCamelCase_	  =					char
      return pairs







class 						A(		UpperCamelCase     ):



 '''simple docstring'''


 UpperCamelCase		     =		VOCAB_FILES_NAMES
 UpperCamelCase		     =		PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase		     =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCamelCase		     =		['''input_ids''', '''attention_mask''']

 def __init__(       self      :						Optional[Any]		,							A_      :						List[Any]		,							A_      :						List[Any]		,							A_      :						Union[str, Any]="replace"		,							A_      :						Dict="<s>"		,							A_      :						Optional[int]="</s>"		,							A_      :						Optional[Any]="</s>"		,							A_      :						Dict="<s>"		,							A_      :						Dict="<unk>"		,							A_      :						Any="<pad>"		,							A_      :						Dict="<mask>"		,							A_      :						Union[str, Any]=False		,							**A_      :						List[str]		,							)     ->			Tuple:

       """simple docstring"""






       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else bos_token
       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else eos_token
       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else sep_token
       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else cls_token
       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else unk_token
       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else mask_token

       super().__init__(
           errors=A_		,							bos_token=A_		,							eos_token=A_		,							unk_token=A_		,							sep_token=A_		,							cls_token=A_		,							pad_token=A_		,							mask_token=A_		,							add_prefix_space=A_		,							**A_		,							)

       with open(A_		,							encoding='utf-8'						) as vocab_handle:
             lowerCamelCase_	  =					json.load(A_						)
       lowerCamelCase_	  =					{v: k for k, v in self.encoder.items()}
       lowerCamelCase_	  =					errors  # how to handle errors in decoding
       lowerCamelCase_	  =					bytes_to_unicode()
       lowerCamelCase_	  =					{v: k for k, v in self.byte_encoder.items()}
       with open(A_		,							encoding='utf-8'						) as merges_handle:
             lowerCamelCase_	  =					merges_handle.read().split('\n'						)[1:-1]
       lowerCamelCase_	  =					[tuple(merge.split()						) for merge in bpe_merges]
       lowerCamelCase_	  =					dict(zip(A_		,							range(len(A_						)						)						)						)
       lowerCamelCase_	  =					{}
       lowerCamelCase_	  =					add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       lowerCamelCase_	  =					re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'						)

 @property
 # Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
 def        a__				(       self      :						Optional[Any]						)     ->			Dict:

       """simple docstring"""






       return len(self.encoder						)

 def        a__				(       self      :						List[Any]						)     ->			Dict:

       """simple docstring"""






       return dict(self.encoder		,							**self.added_tokens_encoder						)

 def        a__				(       self      :						Tuple		,							A_      :						Tuple						)     ->			Optional[Any]:

       """simple docstring"""






       if token in self.cache:
             return self.cache[token]
       lowerCamelCase_	  =					tuple(A_						)
       lowerCamelCase_	  =					get_pairs(A_						)

       if not pairs:
             return token

       while True:
             lowerCamelCase_	  =					min(A_		,							key=lambda A_						: self.bpe_ranks.get(A_		,							float('inf'						)						)						)
             if bigram not in self.bpe_ranks:
                   break
             lowerCamelCase_  ,       lowerCamelCase_	  =					bigram
             lowerCamelCase_	  =					[]
             lowerCamelCase_	  =					0
             while i < len(A_						):
                   try:
                         lowerCamelCase_	  =					word.index(A_		,							A_						)
                   except ValueError:
                         new_word.extend(word[i:]						)
                         break
                   else:
                         new_word.extend(word[i:j]						)
                         lowerCamelCase_	  =					j

                   if word[i] == first and i < len(A_						) - 1 and word[i + 1] == second:
                         new_word.append(first + second						)
                         i += 2
                   else:
                         new_word.append(word[i]						)
                         i += 1
             lowerCamelCase_	  =					tuple(A_						)
             lowerCamelCase_	  =					new_word
             if len(A_						) == 1:
                   break
             else:
                   lowerCamelCase_	  =					get_pairs(A_						)
       lowerCamelCase_	  =					' '.join(A_						)
       lowerCamelCase_	  =					word
       return word

 def        a__				(       self      :						str		,							A_      :						List[str]						)     ->			List[str]:

       """simple docstring"""






       lowerCamelCase_	  =					[]
       for token in re.findall(self.pat		,							A_						):
             lowerCamelCase_	  =					''.join(
                 self.byte_encoder[b] for b in token.encode('utf-8'						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
             bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A_						).split(' '						)						)
       return bpe_tokens

 def        a__				(       self      :						Tuple		,							A_      :						str						)     ->			Optional[Any]:

       """simple docstring"""






       return self.encoder.get(A_		,							self.encoder.get(self.unk_token						)						)

 def        a__				(       self      :						Tuple		,							A_      :						Dict						)     ->			List[Any]:

       """simple docstring"""






       return self.decoder.get(A_						)

 def        a__				(       self      :						Optional[int]		,							A_      :						List[Any]						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					''.join(A_						)
       lowerCamelCase_	  =					bytearray([self.byte_decoder[c] for c in text]						).decode('utf-8'		,							errors=self.errors						)
       return text

 def        a__				(       self      :						Tuple		,							A_      :						str		,							A_      :						Optional[str] = None						)     ->			Tuple[str]:

       """simple docstring"""






       if not os.path.isdir(A_						):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
             return
       lowerCamelCase_	  =					os.path.join(
           A_		,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)
       lowerCamelCase_	  =					os.path.join(
           A_		,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']						)

       with open(A_		,							'w'		,							encoding='utf-8'						) as f:
             f.write(json.dumps(self.encoder		,							indent=2		,							sort_keys=A_		,							ensure_ascii=A_						) + '\n'						)

       lowerCamelCase_	  =					0
       with open(A_		,							'w'		,							encoding='utf-8'						) as writer:
             writer.write('#version: 0.2\n'						)
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,							key=lambda A_						: kv[1]						):
                   if index != token_index:
                         logger.warning(
                             f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                             ' Please check that the tokenizer is not corrupted!'						)
                         lowerCamelCase_	  =					token_index
                   writer.write(' '.join(A_						) + '\n'						)
                   index += 1

       return vocab_file, merge_file

 def        a__				(       self      :						str		,							A_      :						List[int]		,							A_      :						Optional[List[int]] = None		,							A_      :						bool = False						)     ->			List[int]:

       """simple docstring"""






       if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=A_		,							token_ids_a=A_		,							already_has_special_tokens=A_						)

       if token_ids_a is None:
             return [1] + ([0] * len(A_						)) + [1]
       return [1] + ([0] * len(A_						)) + [1, 1] + ([0] * len(A_						)) + [1]

 def        a__				(       self      :						int		,							A_      :						List[int]		,							A_      :						Optional[List[int]] = None						)     ->			List[int]:

       """simple docstring"""






       lowerCamelCase_	  =					[self.sep_token_id]
       lowerCamelCase_	  =					[self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]

 def        a__				(       self      :						str		,							A_      :						Optional[Any]		,							A_      :						Union[str, Any]=False		,							**A_      :						List[str]						)     ->			List[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					kwargs.pop('add_prefix_space'		,							self.add_prefix_space						)
       if (is_split_into_words or add_prefix_space) and (len(A_						) > 0 and not text[0].isspace()):
             lowerCamelCase_	  =					' ' + text
       return (text, kwargs)

 def        a__				(       self      :						List[Any]		,							A_      :						List[int]		,							A_      :						Optional[List[int]] = None						)     ->			Dict:

       """simple docstring"""






       return token_ids_a + [self.eos_token_id]





 def        a__				(       self      :						Optional[int]		,							A_      :						"Conversation"						)     ->			List[int]:

       """simple docstring"""






       lowerCamelCase_	  =					[]
       for is_user, text in conversation.iter_texts():
             if is_user:
                   # We need to space prefix as it's being done within blenderbot
                   inputs.append(' ' + text						)
             else:
                   # Generated responses should contain them already.
                   inputs.append(A_						)

       lowerCamelCase_	  =					'  '.join(A_						)
       lowerCamelCase_	  =					self.encode(A_						)
       if len(A_						) > self.model_max_length:
             lowerCamelCase_	  =					input_ids[-self.model_max_length :]
             logger.warning(f"""Trimmed input from conversation as it was longer than {self.model_max_length} tokens."""						)
       return input_ids



