


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation

__UpperCamelCase				:  Optional[Any]      					=							logging.get_logger(__name__)


__UpperCamelCase				:  int      					=							{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

__UpperCamelCase				:  Optional[Any]      					=							{
    'vocab_file': {'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json'},
    'merges_file': {'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt'},
    'tokenizer_config_file': {
        'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json'
    },
}

__UpperCamelCase				:  List[str]      					=							{'facebook/blenderbot-3B': 128}
@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def      snake_case_						( ):
   UpperCAmelCase_							:      Union[str, Any]						=   (
       list(range(ord('''!'''       )			,       ord('''~'''       ) + 1       )       ) + list(range(ord('''¡'''       )			,       ord('''¬'''       ) + 1       )       ) + list(range(ord('''®'''       )			,       ord('''ÿ'''       ) + 1       )       )
   )
   UpperCAmelCase_							:      Any						=   bs[:]
   UpperCAmelCase_							:      List[Any]						=   0
   for b in range(2**8       ):
      if b not in bs:
         bs.append(__lowercase       )
         cs.append(2**8 + n       )
         n += 1
   UpperCAmelCase_							:      str						=   [chr(__lowercase       ) for n in cs]
   return dict(zip(__lowercase			,       __lowercase       )       )
def      snake_case_						( __lowercase       ):
   UpperCAmelCase_							:      Optional[int]						=   set()
   UpperCAmelCase_							:      Any						=   word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)       )
      UpperCAmelCase_							:      Union[str, Any]						=   char
   return pairs





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     List[Any]			= VOCAB_FILES_NAMES
    A_  :     Any			= PRETRAINED_VOCAB_FILES_MAP
    A_  :     Any			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    A_  :     int			= ['input_ids', 'attention_mask']





    def __init__(	self				:   Optional[int]       ,   __snake_case				:   Any       ,   __snake_case				:   Tuple       ,   __snake_case				:   List[Any]="replace"       ,   __snake_case				:   Union[str, Any]="<s>"       ,   __snake_case				:   Tuple="</s>"       ,   __snake_case				:   Tuple="</s>"       ,   __snake_case				:   Optional[int]="<s>"       ,   __snake_case				:   int="<unk>"       ,   __snake_case				:   List[Any]="<pad>"       ,   __snake_case				:   Tuple="<mask>"       ,   __snake_case				:   Any=False       ,   **__snake_case				:   Dict       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Tuple						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else bos_token
       UpperCAmelCase_							:      str						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else eos_token
       UpperCAmelCase_							:      List[str]						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else sep_token
       UpperCAmelCase_							:      Dict						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else cls_token
       UpperCAmelCase_							:      int						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else unk_token
       UpperCAmelCase_							:      int						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       UpperCAmelCase_							:      List[Any]						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else mask_token

       super().__init__(
           errors=__snake_case       ,   bos_token=__snake_case       ,   eos_token=__snake_case       ,   unk_token=__snake_case       ,   sep_token=__snake_case       ,   cls_token=__snake_case       ,   pad_token=__snake_case       ,   mask_token=__snake_case       ,   add_prefix_space=__snake_case       ,   **__snake_case       ,   )

       with open(__snake_case       ,   encoding='''utf-8'''	) as vocab_handle:
          UpperCAmelCase_							:      str						=   json.load(__snake_case	)
       UpperCAmelCase_							:      List[Any]						=   {v: k for k, v in self.encoder.items()}
       UpperCAmelCase_							:      List[Any]						=   errors  # how to handle errors in decoding
       UpperCAmelCase_							:      Tuple						=   bytes_to_unicode()
       UpperCAmelCase_							:      List[Any]						=   {v: k for k, v in self.byte_encoder.items()}
       with open(__snake_case       ,   encoding='''utf-8'''	) as merges_handle:
          UpperCAmelCase_							:      Dict						=   merges_handle.read().split('''\n'''	)[1:-1]
       UpperCAmelCase_							:      Any						=   [tuple(merge.split()	) for merge in bpe_merges]
       UpperCAmelCase_							:      List[str]						=   dict(zip(__snake_case       ,   range(len(__snake_case	)	)	)	)
       UpperCAmelCase_							:      Tuple						=   {}
       UpperCAmelCase_							:      Union[str, Any]						=   add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       UpperCAmelCase_							:      Optional[Any]						=   re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''	)





    @property
    # Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
    def 					_lowerCamelCase (	self				:   Any	):





       '''simple docstring'''
       return len(self.encoder	)





    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       return dict(self.encoder       ,   **self.added_tokens_encoder	)





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   Dict	):





       '''simple docstring'''
       if token in self.cache:
          return self.cache[token]
       UpperCAmelCase_							:      Optional[Any]						=   tuple(__snake_case	)
       UpperCAmelCase_							:      Optional[int]						=   get_pairs(__snake_case	)

       if not pairs:
          return token

       while True:
          UpperCAmelCase_							:      Dict						=   min(__snake_case       ,   key=lambda __snake_case	: self.bpe_ranks.get(__snake_case       ,   float('''inf'''	)	)	)
          if bigram not in self.bpe_ranks:
             break
          UpperCAmelCase_		,      UpperCAmelCase_							:      str						=   bigram
          UpperCAmelCase_							:      str						=   []
          UpperCAmelCase_							:      List[str]						=   0
          while i < len(__snake_case	):
             try:
                UpperCAmelCase_							:      Dict						=   word.index(__snake_case       ,   __snake_case	)
             except ValueError:
                new_word.extend(word[i:]	)
                break
             else:
                new_word.extend(word[i:j]	)
                UpperCAmelCase_							:      List[Any]						=   j

             if word[i] == first and i < len(__snake_case	) - 1 and word[i + 1] == second:
                new_word.append(first + second	)
                i += 2
             else:
                new_word.append(word[i]	)
                i += 1
          UpperCAmelCase_							:      List[Any]						=   tuple(__snake_case	)
          UpperCAmelCase_							:      str						=   new_word
          if len(__snake_case	) == 1:
             break
          else:
             UpperCAmelCase_							:      Optional[int]						=   get_pairs(__snake_case	)
       UpperCAmelCase_							:      Any						=   ''' '''.join(__snake_case	)
       UpperCAmelCase_							:      str						=   word
       return word





    def 					_lowerCamelCase (	self				:   List[str]       ,   __snake_case				:   Any	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   []
       for token in re.findall(self.pat       ,   __snake_case	):
          UpperCAmelCase_							:      Any						=   ''''''.join(
              self.byte_encoder[b] for b in token.encode('''utf-8'''	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
          bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__snake_case	).split(''' '''	)	)
       return bpe_tokens





    def 					_lowerCamelCase (	self				:   List[Any]       ,   __snake_case				:   List[Any]	):





       '''simple docstring'''
       return self.encoder.get(__snake_case       ,   self.encoder.get(self.unk_token	)	)





    def 					_lowerCamelCase (	self				:   List[str]       ,   __snake_case				:   str	):





       '''simple docstring'''
       return self.decoder.get(__snake_case	)





    def 					_lowerCamelCase (	self				:   Optional[Any]       ,   __snake_case				:   Optional[Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   ''''''.join(__snake_case	)
       UpperCAmelCase_							:      int						=   bytearray([self.byte_decoder[c] for c in text]	).decode('''utf-8'''       ,   errors=self.errors	)
       return text





    def 					_lowerCamelCase (	self				:   int       ,   __snake_case				:   str       ,   __snake_case				:   Optional[str] = None	):





       '''simple docstring'''
       if not os.path.isdir(__snake_case	):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
          return
       UpperCAmelCase_							:      int						=   os.path.join(
           __snake_case       ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)
       UpperCAmelCase_							:      Dict						=   os.path.join(
           __snake_case       ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']	)

       with open(__snake_case       ,   '''w'''       ,   encoding='''utf-8'''	) as f:
          f.write(json.dumps(self.encoder       ,   indent=2       ,   sort_keys=__snake_case       ,   ensure_ascii=__snake_case	) + '''\n'''	)

       UpperCAmelCase_							:      Optional[int]						=   0
       with open(__snake_case       ,   '''w'''       ,   encoding='''utf-8'''	) as writer:
          writer.write('''#version: 0.2\n'''	)
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,   key=lambda __snake_case	: kv[1]	):
             if index != token_index:
                logger.warning(
                    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                    ''' Please check that the tokenizer is not corrupted!'''	)
                UpperCAmelCase_							:      Optional[Any]						=   token_index
             writer.write(''' '''.join(__snake_case	) + '''\n'''	)
             index += 1

       return vocab_file, merge_file





    def 					_lowerCamelCase (	self				:   Optional[int]       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None       ,   __snake_case				:   bool = False	):





       '''simple docstring'''
       if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=__snake_case       ,   token_ids_a=__snake_case       ,   already_has_special_tokens=__snake_case	)

       if token_ids_a is None:
          return [1] + ([0] * len(__snake_case	)) + [1]
       return [1] + ([0] * len(__snake_case	)) + [1, 1] + ([0] * len(__snake_case	)) + [1]





    def 					_lowerCamelCase (	self				:   List[str]       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[str]						=   [self.sep_token_id]
       UpperCAmelCase_							:      int						=   [self.cls_token_id]

       if token_ids_a is None:
          return len(cls + token_ids_a + sep	) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]





    def 					_lowerCamelCase (	self				:   List[Any]       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Any=False       ,   **__snake_case				:   Any	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   kwargs.pop('''add_prefix_space'''       ,   self.add_prefix_space	)
       if (is_split_into_words or add_prefix_space) and (len(__snake_case	) > 0 and not text[0].isspace()):
          UpperCAmelCase_							:      Dict						=   ''' ''' + text
       return (text, kwargs)





    def 					_lowerCamelCase (	self				:   Union[str, Any]       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None	):





       '''simple docstring'''
       return token_ids_a + [self.eos_token_id]





    def 					_lowerCamelCase (	self				:   List[Any]       ,   __snake_case				:   "Conversation"	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   []
       for is_user, text in conversation.iter_texts():
          if is_user:
             # We need to space prefix as it's being done within blenderbot
             inputs.append(''' ''' + text	)
          else:
             # Generated responses should contain them already.
             inputs.append(__snake_case	)

       UpperCAmelCase_							:      List[str]						=   '''  '''.join(__snake_case	)
       UpperCAmelCase_							:      Any						=   self.encode(__snake_case	)
       if len(__snake_case	) > self.model_max_length:
          UpperCAmelCase_							:      Dict						=   input_ids[-self.model_max_length :]
          logger.warning(f'''Trimmed input from conversation as it was longer than {self.model_max_length} tokens.'''	)
       return input_ids



import copy

from ...configuration_utils import PretrainedConfig
from ...utils import add_start_docstrings


__UpperCamelCase				:  Dict      					=							R'\n    [`RagConfig`] stores the configuration of a *RagModel*. Configuration objects inherit from [`PretrainedConfig`] and\n    can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information.\n\n    Args:\n        title_sep (`str`, *optional*, defaults to  `" / "`):\n            Separator inserted between the title and the text of the retrieved document when calling [`RagRetriever`].\n        doc_sep (`str`, *optional*, defaults to  `" // "`):\n            Separator inserted between the text of the retrieved document and the original input when calling\n            [`RagRetriever`].\n        n_docs (`int`, *optional*, defaults to 5):\n            Number of documents to retrieve.\n        max_combined_length (`int`, *optional*, defaults to 300):\n            Max length of contextualized input returned by [`~RagRetriever.__call__`].\n        retrieval_vector_size (`int`, *optional*, defaults to 768):\n            Dimensionality of the document embeddings indexed by [`RagRetriever`].\n        retrieval_batch_size (`int`, *optional*, defaults to 8):\n            Retrieval batch size, defined as the number of queries issues concurrently to the faiss index encapsulated\n            [`RagRetriever`].\n        dataset (`str`, *optional*, defaults to `"wiki_dpr"`):\n            A dataset identifier of the indexed dataset in HuggingFace Datasets (list all available datasets and ids\n            using `datasets.list_datasets()`).\n        dataset_split (`str`, *optional*, defaults to `"train"`)\n            Which split of the `dataset` to load.\n        index_name (`str`, *optional*, defaults to `"compressed"`)\n            The index name of the index associated with the `dataset`. One can choose between `"legacy"`, `"exact"` and\n            `"compressed"`.\n        index_path (`str`, *optional*)\n            The path to the serialized faiss index on disk.\n        passages_path (`str`, *optional*):\n            A path to text passages compatible with the faiss index. Required if using\n            [`~models.rag.retrieval_rag.LegacyIndex`]\n        use_dummy_dataset (`bool`, *optional*, defaults to `False`)\n            Whether to load a "dummy" variant of the dataset specified by `dataset`.\n        label_smoothing (`float`, *optional*, defaults to 0.0):\n            Only relevant if `return_loss` is set to `True`. Controls the `epsilon` parameter value for label smoothing\n            in the loss calculation. If set to 0, no label smoothing is performed.\n        do_marginalize (`bool`, *optional*, defaults to `False`):\n            If `True`, the logits are marginalized over all documents by making use of\n            `torch.nn.functional.log_softmax`.\n        reduce_loss (`bool`, *optional*, defaults to `False`):\n            Whether or not to reduce the NLL loss using the `torch.Tensor.sum` operation.\n        do_deduplication (`bool`, *optional*, defaults to `True`):\n            Whether or not to deduplicate the generations from different context documents for a given input. Has to be\n            set to `False` if used while training with distributed backend.\n        exclude_bos_score (`bool`, *optional*, defaults to `False`):\n            Whether or not to disregard the BOS token when computing the loss.\n        output_retrieved(`bool`, *optional*, defaults to `False`):\n            If set to `True`, `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and\n            `context_attention_mask` are returned. See returned tensors for more detail.\n        use_cache (`bool`, *optional*, defaults to `True`):\n            Whether or not the model should return the last key/values attentions (not used by all models).\n        forced_eos_token_id (`int`, *optional*):\n            The id of the token to force as the last generated token when `max_length` is reached. Usually set to\n            `eos_token_id`.\n'





@add_start_docstrings(snake_case__						)
class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     Union[str, Any]			= 'rag'
    A_  :     Tuple			= True





    def __init__(	self				:   int       ,   __snake_case				:   List[str]=None       ,   __snake_case				:   List[Any]=True       ,   __snake_case				:   Optional[int]=None       ,   __snake_case				:   Union[str, Any]=None       ,   __snake_case				:   int=None       ,   __snake_case				:   Any=None       ,   __snake_case				:   Optional[int]=None       ,   __snake_case				:   Optional[int]=" / "       ,   __snake_case				:   Any=" // "       ,   __snake_case				:   Tuple=5       ,   __snake_case				:   Union[str, Any]=300       ,   __snake_case				:   Any=768       ,   __snake_case				:   Tuple=8       ,   __snake_case				:   int="wiki_dpr"       ,   __snake_case				:   Optional[int]="train"       ,   __snake_case				:   Tuple="compressed"       ,   __snake_case				:   Optional[int]=None       ,   __snake_case				:   List[Any]=None       ,   __snake_case				:   Optional[int]=False       ,   __snake_case				:   str=False       ,   __snake_case				:   Dict=0.0       ,   __snake_case				:   Union[str, Any]=True       ,   __snake_case				:   Union[str, Any]=False       ,   __snake_case				:   str=False       ,   __snake_case				:   str=False       ,   __snake_case				:   Optional[Any]=True       ,   __snake_case				:   int=None       ,   **__snake_case				:   str       ,   ):





       '''simple docstring'''
       super().__init__(
           bos_token_id=__snake_case       ,   pad_token_id=__snake_case       ,   eos_token_id=__snake_case       ,   decoder_start_token_id=__snake_case       ,   forced_eos_token_id=__snake_case       ,   is_encoder_decoder=__snake_case       ,   prefix=__snake_case       ,   vocab_size=__snake_case       ,   **__snake_case       ,   )
       assert (
           "question_encoder" in kwargs and "generator" in kwargs
       ), "Config has to be initialized with question_encoder and generator config"
       UpperCAmelCase_							:      Dict						=   kwargs.pop('''question_encoder'''	)
       UpperCAmelCase_							:      List[Any]						=   question_encoder_config.pop('''model_type'''	)
       UpperCAmelCase_							:      Any						=   kwargs.pop('''generator'''	)
       UpperCAmelCase_							:      Union[str, Any]						=   decoder_config.pop('''model_type'''	)

       from ..auto.configuration_auto import AutoConfig

       UpperCAmelCase_							:      Any						=   AutoConfig.for_model(__snake_case       ,   **__snake_case	)
       UpperCAmelCase_							:      int						=   AutoConfig.for_model(__snake_case       ,   **__snake_case	)

       UpperCAmelCase_							:      Any						=   reduce_loss
       UpperCAmelCase_							:      List[Any]						=   label_smoothing
       UpperCAmelCase_							:      Tuple						=   exclude_bos_score
       UpperCAmelCase_							:      int						=   do_marginalize

       UpperCAmelCase_							:      Tuple						=   title_sep
       UpperCAmelCase_							:      Union[str, Any]						=   doc_sep
       UpperCAmelCase_							:      Any						=   n_docs
       UpperCAmelCase_							:      Optional[int]						=   max_combined_length

       UpperCAmelCase_							:      Any						=   dataset
       UpperCAmelCase_							:      List[Any]						=   dataset_split
       UpperCAmelCase_							:      Union[str, Any]						=   index_name

       UpperCAmelCase_							:      List[str]						=   retrieval_vector_size
       UpperCAmelCase_							:      Optional[Any]						=   retrieval_batch_size
       UpperCAmelCase_							:      Optional[int]						=   passages_path
       UpperCAmelCase_							:      Optional[Any]						=   index_path
       UpperCAmelCase_							:      List[Any]						=   use_dummy_dataset

       UpperCAmelCase_							:      int						=   output_retrieved

       UpperCAmelCase_							:      int						=   do_deduplication

       UpperCAmelCase_							:      Optional[int]						=   use_cache

       if self.forced_eos_token_id is None:
          UpperCAmelCase_							:      int						=   getattr(self.generator       ,   '''forced_eos_token_id'''       ,   __snake_case	)





    @classmethod
    def 					_lowerCamelCase (	cls				:   Optional[Any]       ,   __snake_case				:   PretrainedConfig       ,   __snake_case				:   PretrainedConfig       ,   **__snake_case				:   str	):





       '''simple docstring'''
       return cls(question_encoder=question_encoder_config.to_dict()       ,   generator=generator_config.to_dict()       ,   **__snake_case	)





    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   copy.deepcopy(self.__dict__	)
       UpperCAmelCase_							:      Optional[int]						=   self.question_encoder.to_dict()
       UpperCAmelCase_							:      Dict						=   self.generator.to_dict()
       UpperCAmelCase_							:      Optional[Any]						=   self.__class__.model_type
       return output