



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation

a__      :				List[str]   	=			logging.get_logger(__name__)


a__      :				int   	=			{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

a__      :				List[str]   	=			{
    "vocab_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"},
    "merges_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"},
    "tokenizer_config_file": {
        "facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"
    },
}

a__      :				Dict   	=			{"facebook/blenderbot-3B": 1_28}

@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def   _lowerCAmelCase   (						):
      lowercase__     =  (
          list(range(ord('!'	)						,  ord('~'	) + 1	)	) + list(range(ord('¡'	)						,  ord('¬'	) + 1	)	) + list(range(ord('®'	)						,  ord('ÿ'	) + 1	)	)
      )
      lowercase__     =  bs[:]
      lowercase__     =  0
      for b in range(2**8	):
            if b not in bs:
                  bs.append(A__	)
                  cs.append(2**8 + n	)
                  n += 1
      lowercase__     =  [chr(A__	) for n in cs]
      return dict(zip(A__						,  A__	)	)

def   _lowerCAmelCase   (						A__	):
      lowercase__     =  set()
      lowercase__     =  word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)	)
            lowercase__     =  char
      return pairs



class       UpperCAmelCase__(							lowerCamelCase	):

   '''simple docstring'''




   A    :  List[Any]       				=    VOCAB_FILES_NAMES
   A    :  Tuple       				=    PRETRAINED_VOCAB_FILES_MAP
   A    :  int       				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A    :  Tuple       				=    ["input_ids", "attention_mask"]


   def __init__(						self  :			Dict			,				lowerCAmelCase  :			Dict			,				lowerCAmelCase  :			List[str]			,				lowerCAmelCase  :			List[str]="replace"			,				lowerCAmelCase  :			List[Any]="<s>"			,				lowerCAmelCase  :			int="</s>"			,				lowerCAmelCase  :			Tuple="</s>"			,				lowerCAmelCase  :			Tuple="<s>"			,				lowerCAmelCase  :			List[Any]="<unk>"			,				lowerCAmelCase  :			Optional[Any]="<pad>"			,				lowerCAmelCase  :			Union[str, Any]="<mask>"			,				lowerCAmelCase  :			Any=False			,				**lowerCAmelCase  :			List[str]			,				)	->				Optional[Any]:




         """simple docstring"""

         lowercase__     =  AddedToken(lowerCAmelCase			,				lstrip=lowerCAmelCase			,				rstrip=lowerCAmelCase) if isinstance(lowerCAmelCase			,				lowerCAmelCase) else bos_token
         lowercase__     =  AddedToken(lowerCAmelCase			,				lstrip=lowerCAmelCase			,				rstrip=lowerCAmelCase) if isinstance(lowerCAmelCase			,				lowerCAmelCase) else eos_token
         lowercase__     =  AddedToken(lowerCAmelCase			,				lstrip=lowerCAmelCase			,				rstrip=lowerCAmelCase) if isinstance(lowerCAmelCase			,				lowerCAmelCase) else sep_token
         lowercase__     =  AddedToken(lowerCAmelCase			,				lstrip=lowerCAmelCase			,				rstrip=lowerCAmelCase) if isinstance(lowerCAmelCase			,				lowerCAmelCase) else cls_token
         lowercase__     =  AddedToken(lowerCAmelCase			,				lstrip=lowerCAmelCase			,				rstrip=lowerCAmelCase) if isinstance(lowerCAmelCase			,				lowerCAmelCase) else unk_token
         lowercase__     =  AddedToken(lowerCAmelCase			,				lstrip=lowerCAmelCase			,				rstrip=lowerCAmelCase) if isinstance(lowerCAmelCase			,				lowerCAmelCase) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         lowercase__     =  AddedToken(lowerCAmelCase			,				lstrip=lowerCAmelCase			,				rstrip=lowerCAmelCase) if isinstance(lowerCAmelCase			,				lowerCAmelCase) else mask_token

         super().__init__(
             errors=lowerCAmelCase			,				bos_token=lowerCAmelCase			,				eos_token=lowerCAmelCase			,				unk_token=lowerCAmelCase			,				sep_token=lowerCAmelCase			,				cls_token=lowerCAmelCase			,				pad_token=lowerCAmelCase			,				mask_token=lowerCAmelCase			,				add_prefix_space=lowerCAmelCase			,				**lowerCAmelCase			,				)

         with open(lowerCAmelCase			,				encoding='utf-8') as vocab_handle:
               lowercase__     =  json.load(lowerCAmelCase)
         lowercase__     =  {v: k for k, v in self.encoder.items()}
         lowercase__     =  errors  # how to handle errors in decoding
         lowercase__     =  bytes_to_unicode()
         lowercase__     =  {v: k for k, v in self.byte_encoder.items()}
         with open(lowerCAmelCase			,				encoding='utf-8') as merges_handle:
               lowercase__     =  merges_handle.read().split('\n')[1:-1]
         lowercase__     =  [tuple(merge.split()) for merge in bpe_merges]
         lowercase__     =  dict(zip(lowerCAmelCase			,				range(len(lowerCAmelCase))))
         lowercase__     =  {}
         lowercase__     =  add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         lowercase__     =  re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+')


   @property
   # Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
   def 			UpperCAmelCase		(						self  :			List[Any])	->				Tuple:




         """simple docstring"""

         return len(self.encoder)


   def 			UpperCAmelCase		(						self  :			List[Any])	->				Optional[int]:




         """simple docstring"""

         return dict(self.encoder			,				**self.added_tokens_encoder)


   def 			UpperCAmelCase		(						self  :			Optional[int]			,				lowerCAmelCase  :			int)	->				Union[str, Any]:




         """simple docstring"""

         if token in self.cache:
               return self.cache[token]
         lowercase__     =  tuple(lowerCAmelCase)
         lowercase__     =  get_pairs(lowerCAmelCase)

         if not pairs:
               return token

         while True:
               lowercase__     =  min(lowerCAmelCase			,				key=lambda lowerCAmelCase: self.bpe_ranks.get(lowerCAmelCase			,				float('inf')))
               if bigram not in self.bpe_ranks:
                     break
               lowercase__,	lowercase__     =  bigram
               lowercase__     =  []
               lowercase__     =  0
               while i < len(lowerCAmelCase):
                     try:
                           lowercase__     =  word.index(lowerCAmelCase			,				lowerCAmelCase)
                     except ValueError:
                           new_word.extend(word[i:])
                           break
                     else:
                           new_word.extend(word[i:j])
                           lowercase__     =  j

                     if word[i] == first and i < len(lowerCAmelCase) - 1 and word[i + 1] == second:
                           new_word.append(first + second)
                           i += 2
                     else:
                           new_word.append(word[i])
                           i += 1
               lowercase__     =  tuple(lowerCAmelCase)
               lowercase__     =  new_word
               if len(lowerCAmelCase) == 1:
                     break
               else:
                     lowercase__     =  get_pairs(lowerCAmelCase)
         lowercase__     =  ' '.join(lowerCAmelCase)
         lowercase__     =  word
         return word


   def 			UpperCAmelCase		(						self  :			Union[str, Any]			,				lowerCAmelCase  :			str)	->				Optional[Any]:




         """simple docstring"""

         lowercase__     =  []
         for token in re.findall(self.pat			,				lowerCAmelCase):
               lowercase__     =  ''.join(
                   self.byte_encoder[b] for b in token.encode('utf-8'))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase).split(' '))
         return bpe_tokens


   def 			UpperCAmelCase		(						self  :			Optional[Any]			,				lowerCAmelCase  :			str)	->				Union[str, Any]:




         """simple docstring"""

         return self.encoder.get(lowerCAmelCase			,				self.encoder.get(self.unk_token))


   def 			UpperCAmelCase		(						self  :			Union[str, Any]			,				lowerCAmelCase  :			str)	->				Optional[int]:




         """simple docstring"""

         return self.decoder.get(lowerCAmelCase)


   def 			UpperCAmelCase		(						self  :			Optional[int]			,				lowerCAmelCase  :			List[Any])	->				Dict:




         """simple docstring"""

         lowercase__     =  ''.join(lowerCAmelCase)
         lowercase__     =  bytearray([self.byte_decoder[c] for c in text]).decode('utf-8'			,				errors=self.errors)
         return text


   def 			UpperCAmelCase		(						self  :			Any			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			Optional[str] = None)	->				Tuple[str]:




         """simple docstring"""

         if not os.path.isdir(lowerCAmelCase):
               logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
               return
         lowercase__     =  os.path.join(
             lowerCAmelCase			,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
         lowercase__     =  os.path.join(
             lowerCAmelCase			,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

         with open(lowerCAmelCase			,				'w'			,				encoding='utf-8') as f:
               f.write(json.dumps(self.encoder			,				indent=2			,				sort_keys=lowerCAmelCase			,				ensure_ascii=lowerCAmelCase) + '\n')

         lowercase__     =  0
         with open(lowerCAmelCase			,				'w'			,				encoding='utf-8') as writer:
               writer.write('#version: 0.2\n')
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,				key=lambda lowerCAmelCase: kv[1]):
                     if index != token_index:
                           logger.warning(
                               f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                               ' Please check that the tokenizer is not corrupted!')
                           lowercase__     =  token_index
                     writer.write(' '.join(lowerCAmelCase) + '\n')
                     index += 1

         return vocab_file, merge_file


   def 			UpperCAmelCase		(						self  :			Dict			,				lowerCAmelCase  :			List[int]			,				lowerCAmelCase  :			Optional[List[int]] = None			,				lowerCAmelCase  :			bool = False)	->				List[int]:




         """simple docstring"""

         if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=lowerCAmelCase			,				token_ids_a=lowerCAmelCase			,				already_has_special_tokens=lowerCAmelCase)

         if token_ids_a is None:
               return [1] + ([0] * len(lowerCAmelCase)) + [1]
         return [1] + ([0] * len(lowerCAmelCase)) + [1, 1] + ([0] * len(lowerCAmelCase)) + [1]


   def 			UpperCAmelCase		(						self  :			Dict			,				lowerCAmelCase  :			List[int]			,				lowerCAmelCase  :			Optional[List[int]] = None)	->				List[int]:




         """simple docstring"""

         lowercase__     =  [self.sep_token_id]
         lowercase__     =  [self.cls_token_id]

         if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]


   def 			UpperCAmelCase		(						self  :			Tuple			,				lowerCAmelCase  :			Union[str, Any]			,				lowerCAmelCase  :			Tuple=False			,				**lowerCAmelCase  :			Union[str, Any])	->				Tuple:




         """simple docstring"""

         lowercase__     =  kwargs.pop('add_prefix_space'			,				self.add_prefix_space)
         if (is_split_into_words or add_prefix_space) and (len(lowerCAmelCase) > 0 and not text[0].isspace()):
               lowercase__     =  ' ' + text
         return (text, kwargs)


   def 			UpperCAmelCase		(						self  :			str			,				lowerCAmelCase  :			List[int]			,				lowerCAmelCase  :			Optional[List[int]] = None)	->				Optional[Any]:




         """simple docstring"""

         return token_ids_a + [self.eos_token_id]



   def 			UpperCAmelCase		(						self  :			List[str]			,				lowerCAmelCase  :			"Conversation")	->				List[int]:




         """simple docstring"""

         lowercase__     =  []
         for is_user, text in conversation.iter_texts():
               if is_user:
                     # We need to space prefix as it's being done within blenderbot
                     inputs.append(' ' + text)
               else:
                     # Generated responses should contain them already.
                     inputs.append(lowerCAmelCase)

         lowercase__     =  '  '.join(lowerCAmelCase)
         lowercase__     =  self.encode(lowerCAmelCase)
         if len(lowerCAmelCase) > self.model_max_length:
               lowercase__     =  input_ids[-self.model_max_length :]
               logger.warning(f'''Trimmed input from conversation as it was longer than {self.model_max_length} tokens.''')
         return input_ids





import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


a__      :				List[Any]   	=			logging.get_logger(__name__)


a__      :				Optional[int]   	=			{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

# See all BART models at https://huggingface.co/models?filter=bart
a__      :				List[Any]   	=			{
    "vocab_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/vocab.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/vocab.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json",
    },
    "merges_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/merges.txt",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/merges.txt",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json",
    },
}

a__      :				int   	=			{
    "facebook/bart-base": 10_24,
    "facebook/bart-large": 10_24,
    "facebook/bart-large-mnli": 10_24,
    "facebook/bart-large-cnn": 10_24,
    "facebook/bart-large-xsum": 10_24,
    "yjernite/bart_eli5": 10_24,
}



class       UpperCAmelCase__(							lowerCamelCase	):

   '''simple docstring'''




   A    :  Optional[Any]       				=    VOCAB_FILES_NAMES
   A    :  Dict       				=    PRETRAINED_VOCAB_FILES_MAP
   A    :  str       				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A    :  int       				=    ["input_ids", "attention_mask"]
   A    :  Any       				=    BartTokenizer


   def __init__(						self  :			List[Any]			,				lowerCAmelCase  :			Any=None			,				lowerCAmelCase  :			Optional[int]=None			,				lowerCAmelCase  :			List[Any]=None			,				lowerCAmelCase  :			str="replace"			,				lowerCAmelCase  :			str="<s>"			,				lowerCAmelCase  :			int="</s>"			,				lowerCAmelCase  :			Optional[int]="</s>"			,				lowerCAmelCase  :			Union[str, Any]="<s>"			,				lowerCAmelCase  :			str="<unk>"			,				lowerCAmelCase  :			int="<pad>"			,				lowerCAmelCase  :			int="<mask>"			,				lowerCAmelCase  :			Dict=False			,				lowerCAmelCase  :			List[Any]=True			,				**lowerCAmelCase  :			Optional[Any]			,				)	->				Optional[Any]:




         """simple docstring"""

         super().__init__(
             lowerCAmelCase			,				lowerCAmelCase			,				tokenizer_file=lowerCAmelCase			,				errors=lowerCAmelCase			,				bos_token=lowerCAmelCase			,				eos_token=lowerCAmelCase			,				sep_token=lowerCAmelCase			,				cls_token=lowerCAmelCase			,				unk_token=lowerCAmelCase			,				pad_token=lowerCAmelCase			,				mask_token=lowerCAmelCase			,				add_prefix_space=lowerCAmelCase			,				trim_offsets=lowerCAmelCase			,				**lowerCAmelCase			,				)

         lowercase__     =  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
         if pre_tok_state.get('add_prefix_space'			,				lowerCAmelCase) != add_prefix_space:
               lowercase__     =  getattr(lowerCAmelCase			,				pre_tok_state.pop('type'))
               lowercase__     =  add_prefix_space
               lowercase__     =  pre_tok_class(**lowerCAmelCase)

         lowercase__     =  add_prefix_space

         # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
         lowercase__     =  'post_processor'
         lowercase__     =  getattr(self.backend_tokenizer			,				lowerCAmelCase			,				lowerCAmelCase)
         if tokenizer_component_instance:
               lowercase__     =  json.loads(tokenizer_component_instance.__getstate__())

               # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
               if "sep" in state:
                     lowercase__     =  tuple(state['sep'])
               if "cls" in state:
                     lowercase__     =  tuple(state['cls'])

               lowercase__     =  False

               if state.get('add_prefix_space'			,				lowerCAmelCase) != add_prefix_space:
                     lowercase__     =  add_prefix_space
                     lowercase__     =  True

               if state.get('trim_offsets'			,				lowerCAmelCase) != trim_offsets:
                     lowercase__     =  trim_offsets
                     lowercase__     =  True

               if changes_to_apply:
                     lowercase__     =  getattr(lowerCAmelCase			,				state.pop('type'))
                     lowercase__     =  component_class(**lowerCAmelCase)
                     setattr(self.backend_tokenizer			,				lowerCAmelCase			,				lowerCAmelCase)


   @property
   def 			UpperCAmelCase		(						self  :			Union[str, Any])	->				str:




         """simple docstring"""

         if self._mask_token is None:
               if self.verbose:
                     logger.error('Using mask_token, but it is not set yet.')
               return None
         return str(self._mask_token)


   @mask_token.setter
   def 			UpperCAmelCase		(						self  :			Tuple			,				lowerCAmelCase  :			int)	->				Optional[int]:




         """simple docstring"""

         lowercase__     =  AddedToken(lowerCAmelCase			,				lstrip=lowerCAmelCase			,				rstrip=lowerCAmelCase) if isinstance(lowerCAmelCase			,				lowerCAmelCase) else value
         lowercase__     =  value


   def 			UpperCAmelCase		(						self  :			List[str]			,				*lowerCAmelCase  :			Tuple			,				**lowerCAmelCase  :			Optional[int])	->				BatchEncoding:




         """simple docstring"""

         lowercase__     =  kwargs.get('is_split_into_words'			,				lowerCAmelCase)

         if is_split_into_words and not self.add_prefix_space:
               raise ValueError(
                   f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                   'to use it with pretokenized inputs.')

         return super()._batch_encode_plus(*lowerCAmelCase			,				**lowerCAmelCase)


   def 			UpperCAmelCase		(						self  :			str			,				*lowerCAmelCase  :			Tuple			,				**lowerCAmelCase  :			str)	->				BatchEncoding:




         """simple docstring"""

         lowercase__     =  kwargs.get('is_split_into_words'			,				lowerCAmelCase)

         if is_split_into_words and not self.add_prefix_space:
               raise ValueError(
                   f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                   'to use it with pretokenized inputs.')

         return super()._encode_plus(*lowerCAmelCase			,				**lowerCAmelCase)


   def 			UpperCAmelCase		(						self  :			Dict			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			Optional[str] = None)	->				Tuple[str]:




         """simple docstring"""

         lowercase__     =  self._tokenizer.model.save(lowerCAmelCase			,				name=lowerCAmelCase)
         return tuple(lowerCAmelCase)


   def 			UpperCAmelCase		(						self  :			Any			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			Optional[int]=None)	->				Tuple:




         """simple docstring"""

         lowercase__     =  [self.bos_token_id] + token_ids_a + [self.eos_token_id]
         if token_ids_a is None:
               return output

         return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



   def 			UpperCAmelCase		(						self  :			Union[str, Any]			,				lowerCAmelCase  :			List[int]			,				lowerCAmelCase  :			Optional[List[int]] = None)	->				List[int]:




         """simple docstring"""

         lowercase__     =  [self.sep_token_id]
         lowercase__     =  [self.cls_token_id]

         if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]
