





'''simple docstring'''







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation

snake_case_     :       Optional[Any]						=  logging.get_logger(__name__)


snake_case_     :       str						=  {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

snake_case_     :       Any						=  {
    "vocab_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"},
    "merges_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"},
    "tokenizer_config_file": {
        "facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"
    },
}

snake_case_     :       List[Any]						=  {"facebook/blenderbot-3B": 1_28}





@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def        lowerCamelCase_      (  ) ->	List[Any]:
						UpperCAmelCase_			:      Any     						=  (
						    list(range(ord('''!'''   ),			ord('''~'''   ) + 1   )   ) + list(range(ord('''¡'''   ),			ord('''¬'''   ) + 1   )   ) + list(range(ord('''®'''   ),			ord('''ÿ'''   ) + 1   )   )
						)
						UpperCAmelCase_			:      Optional[int]     						=  bs[:]
						UpperCAmelCase_			:      List[Any]     						=  0
						for b in range(2**8   ):
												if b not in bs:
																		bs.append(SCREAMING_SNAKE_CASE__   )
																		cs.append(2**8 + n   )
																		n += 1
						UpperCAmelCase_			:      Dict     						=  [chr(SCREAMING_SNAKE_CASE__   ) for n in cs]
						return dict(zip(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )   )





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			Any   ) ->	str:
						UpperCAmelCase_			:      List[Any]     						=  set()
						UpperCAmelCase_			:      Any     						=  word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)   )
												UpperCAmelCase_			:      Optional[int]     						=  char
						return pairs


class 					__a      (lowerCamelCase     ):
							__a					:   List[Any]									=       VOCAB_FILES_NAMES
							__a					:   Optional[Any]									=       PRETRAINED_VOCAB_FILES_MAP
							__a					:   Optional[int]									=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a					:   List[Any]									=       ["input_ids", "attention_mask"]

							def __init__(							self							:     int       ,     __magic_name__							:     List[str]       ,     __magic_name__							:     Any       ,     __magic_name__							:     str="replace"       ,     __magic_name__							:     Optional[Any]="<s>"       ,     __magic_name__							:     List[Any]="</s>"       ,     __magic_name__							:     Any="</s>"       ,     __magic_name__							:     List[str]="<s>"       ,     __magic_name__							:     Union[str, Any]="<unk>"       ,     __magic_name__							:     List[str]="<pad>"       ,     __magic_name__							:     str="<mask>"       ,     __magic_name__							:     Any=False       ,     **__magic_name__							:     Tuple       ,     )					->       Tuple:







													"""simple docstring"""
													UpperCAmelCase_			:      int     						=  AddedToken(__magic_name__       ,     lstrip=__magic_name__       ,     rstrip=__magic_name__   ) if isinstance(__magic_name__       ,     __magic_name__   ) else bos_token
													UpperCAmelCase_			:      int     						=  AddedToken(__magic_name__       ,     lstrip=__magic_name__       ,     rstrip=__magic_name__   ) if isinstance(__magic_name__       ,     __magic_name__   ) else eos_token
													UpperCAmelCase_			:      str     						=  AddedToken(__magic_name__       ,     lstrip=__magic_name__       ,     rstrip=__magic_name__   ) if isinstance(__magic_name__       ,     __magic_name__   ) else sep_token
													UpperCAmelCase_			:      Tuple     						=  AddedToken(__magic_name__       ,     lstrip=__magic_name__       ,     rstrip=__magic_name__   ) if isinstance(__magic_name__       ,     __magic_name__   ) else cls_token
													UpperCAmelCase_			:      Optional[int]     						=  AddedToken(__magic_name__       ,     lstrip=__magic_name__       ,     rstrip=__magic_name__   ) if isinstance(__magic_name__       ,     __magic_name__   ) else unk_token
													UpperCAmelCase_			:      Dict     						=  AddedToken(__magic_name__       ,     lstrip=__magic_name__       ,     rstrip=__magic_name__   ) if isinstance(__magic_name__       ,     __magic_name__   ) else pad_token

													# Mask token behave like a normal word, i.e. include the space before it
													UpperCAmelCase_			:      Dict     						=  AddedToken(__magic_name__       ,     lstrip=__magic_name__       ,     rstrip=__magic_name__   ) if isinstance(__magic_name__       ,     __magic_name__   ) else mask_token

													super().__init__(
													    errors=__magic_name__       ,     bos_token=__magic_name__       ,     eos_token=__magic_name__       ,     unk_token=__magic_name__       ,     sep_token=__magic_name__       ,     cls_token=__magic_name__       ,     pad_token=__magic_name__       ,     mask_token=__magic_name__       ,     add_prefix_space=__magic_name__       ,     **__magic_name__       ,     )

													with open(__magic_name__       ,     encoding='''utf-8'''   ) as vocab_handle:
																			UpperCAmelCase_			:      Dict     						=  json.load(__magic_name__   )
													UpperCAmelCase_			:      List[Any]     						=  {v: k for k, v in self.encoder.items()}
													UpperCAmelCase_			:      Any     						=  errors  # how to handle errors in decoding
													UpperCAmelCase_			:      Optional[int]     						=  bytes_to_unicode()
													UpperCAmelCase_			:      List[Any]     						=  {v: k for k, v in self.byte_encoder.items()}
													with open(__magic_name__       ,     encoding='''utf-8'''   ) as merges_handle:
																			UpperCAmelCase_			:      str     						=  merges_handle.read().split('''\n'''   )[1:-1]
													UpperCAmelCase_			:      List[Any]     						=  [tuple(merge.split()   ) for merge in bpe_merges]
													UpperCAmelCase_			:      Tuple     						=  dict(zip(__magic_name__       ,     range(len(__magic_name__   )   )   )   )
													UpperCAmelCase_			:      List[str]     						=  {}
													UpperCAmelCase_			:      Union[str, Any]     						=  add_prefix_space

													# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
													UpperCAmelCase_			:      Any     						=  re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''   )

							@property
							# Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
							def     UpperCAmelCase__	(							self							:     Optional[Any]   )					->       str:







													"""simple docstring"""
													return len(self.encoder   )

							def     UpperCAmelCase__	(							self							:     Dict   )					->       Dict:







													"""simple docstring"""
													return dict(self.encoder       ,     **self.added_tokens_encoder   )

							def     UpperCAmelCase__	(							self							:     int       ,     __magic_name__							:     List[str]   )					->       str:







													"""simple docstring"""
													if token in self.cache:
																			return self.cache[token]
													UpperCAmelCase_			:      List[Any]     						=  tuple(__magic_name__   )
													UpperCAmelCase_			:      int     						=  get_pairs(__magic_name__   )

													if not pairs:
																			return token

													while True:
																			UpperCAmelCase_			:      Any     						=  min(__magic_name__       ,     key=lambda __magic_name__   : self.bpe_ranks.get(__magic_name__       ,     float('''inf'''   )   )   )
																			if bigram not in self.bpe_ranks:
																									break
																			UpperCAmelCase_						,       UpperCAmelCase_			:      Optional[int]     						=  bigram
																			UpperCAmelCase_			:      int     						=  []
																			UpperCAmelCase_			:      List[str]     						=  0
																			while i < len(__magic_name__   ):
																									try:
																															UpperCAmelCase_			:      Any     						=  word.index(__magic_name__       ,     __magic_name__   )
																									except ValueError:
																															new_word.extend(word[i:]   )
																															break
																									else:
																															new_word.extend(word[i:j]   )
																															UpperCAmelCase_			:      Optional[int]     						=  j

																									if word[i] == first and i < len(__magic_name__   ) - 1 and word[i + 1] == second:
																															new_word.append(first + second   )
																															i += 2
																									else:
																															new_word.append(word[i]   )
																															i += 1
																			UpperCAmelCase_			:      Union[str, Any]     						=  tuple(__magic_name__   )
																			UpperCAmelCase_			:      List[Any]     						=  new_word
																			if len(__magic_name__   ) == 1:
																									break
																			else:
																									UpperCAmelCase_			:      Any     						=  get_pairs(__magic_name__   )
													UpperCAmelCase_			:      Optional[int]     						=  ''' '''.join(__magic_name__   )
													UpperCAmelCase_			:      Union[str, Any]     						=  word
													return word

							def     UpperCAmelCase__	(							self							:     int       ,     __magic_name__							:     List[Any]   )					->       Optional[int]:







													"""simple docstring"""
													UpperCAmelCase_			:      Any     						=  []
													for token in re.findall(self.pat       ,     __magic_name__   ):
																			UpperCAmelCase_			:      Tuple     						=  ''''''.join(
																			    self.byte_encoder[b] for b in token.encode('''utf-8'''   )   )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																			bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__magic_name__   ).split(''' '''   )   )
													return bpe_tokens

							def     UpperCAmelCase__	(							self							:     str       ,     __magic_name__							:     str   )					->       str:







													"""simple docstring"""
													return self.encoder.get(__magic_name__       ,     self.encoder.get(self.unk_token   )   )

							def     UpperCAmelCase__	(							self							:     List[str]       ,     __magic_name__							:     List[Any]   )					->       List[Any]:







													"""simple docstring"""
													return self.decoder.get(__magic_name__   )

							def     UpperCAmelCase__	(							self							:     int       ,     __magic_name__							:     Tuple   )					->       str:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[Any]     						=  ''''''.join(__magic_name__   )
													UpperCAmelCase_			:      Tuple     						=  bytearray([self.byte_decoder[c] for c in text]   ).decode('''utf-8'''       ,     errors=self.errors   )
													return text

							def     UpperCAmelCase__	(							self							:     Optional[int]       ,     __magic_name__							:     str       ,     __magic_name__							:     Optional[str] = None   )					->       Tuple[str]:







													"""simple docstring"""
													if not os.path.isdir(__magic_name__   ):
																			logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
																			return
													UpperCAmelCase_			:      List[Any]     						=  os.path.join(
													    __magic_name__       ,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
													UpperCAmelCase_			:      Tuple     						=  os.path.join(
													    __magic_name__       ,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']   )

													with open(__magic_name__       ,     '''w'''       ,     encoding='''utf-8'''   ) as f:
																			f.write(json.dumps(self.encoder       ,     indent=2       ,     sort_keys=__magic_name__       ,     ensure_ascii=__magic_name__   ) + '''\n'''   )

													UpperCAmelCase_			:      int     						=  0
													with open(__magic_name__       ,     '''w'''       ,     encoding='''utf-8'''   ) as writer:
																			writer.write('''#version: 0.2\n'''   )
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,     key=lambda __magic_name__   : kv[1]   ):
																									if index != token_index:
																															logger.warning(
																															    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																															    ''' Please check that the tokenizer is not corrupted!'''   )
																															UpperCAmelCase_			:      Optional[int]     						=  token_index
																									writer.write(''' '''.join(__magic_name__   ) + '''\n'''   )
																									index += 1

													return vocab_file, merge_file

							def     UpperCAmelCase__	(							self							:     str       ,     __magic_name__							:     List[int]       ,     __magic_name__							:     Optional[List[int]] = None       ,     __magic_name__							:     bool = False   )					->       List[int]:







													"""simple docstring"""
													if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=__magic_name__       ,     token_ids_a=__magic_name__       ,     already_has_special_tokens=__magic_name__   )

													if token_ids_a is None:
																			return [1] + ([0] * len(__magic_name__   )) + [1]
													return [1] + ([0] * len(__magic_name__   )) + [1, 1] + ([0] * len(__magic_name__   )) + [1]

							def     UpperCAmelCase__	(							self							:     Union[str, Any]       ,     __magic_name__							:     List[int]       ,     __magic_name__							:     Optional[List[int]] = None   )					->       List[int]:







													"""simple docstring"""
													UpperCAmelCase_			:      Any     						=  [self.sep_token_id]
													UpperCAmelCase_			:      Optional[Any]     						=  [self.cls_token_id]

													if token_ids_a is None:
																			return len(cls + token_ids_a + sep   ) * [0]
													return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]

							def     UpperCAmelCase__	(							self							:     List[Any]       ,     __magic_name__							:     List[Any]       ,     __magic_name__							:     str=False       ,     **__magic_name__							:     Optional[Any]   )					->       int:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[Any]     						=  kwargs.pop('''add_prefix_space'''       ,     self.add_prefix_space   )
													if (is_split_into_words or add_prefix_space) and (len(__magic_name__   ) > 0 and not text[0].isspace()):
																			UpperCAmelCase_			:      List[str]     						=  ''' ''' + text
													return (text, kwargs)

							def     UpperCAmelCase__	(							self							:     Tuple       ,     __magic_name__							:     List[int]       ,     __magic_name__							:     Optional[List[int]] = None   )					->       List[Any]:







													"""simple docstring"""
													return token_ids_a + [self.eos_token_id]



							def     UpperCAmelCase__	(							self							:     Dict       ,     __magic_name__							:     "Conversation"   )					->       List[int]:







													"""simple docstring"""
													UpperCAmelCase_			:      Union[str, Any]     						=  []
													for is_user, text in conversation.iter_texts():
																			if is_user:
																									# We need to space prefix as it's being done within blenderbot
																									inputs.append(''' ''' + text   )
																			else:
																									# Generated responses should contain them already.
																									inputs.append(__magic_name__   )

													UpperCAmelCase_			:      Optional[int]     						=  '''  '''.join(__magic_name__   )
													UpperCAmelCase_			:      Union[str, Any]     						=  self.encode(__magic_name__   )
													if len(__magic_name__   ) > self.model_max_length:
																			UpperCAmelCase_			:      Tuple     						=  input_ids[-self.model_max_length :]
																			logger.warning(F"""Trimmed input from conversation as it was longer than {self.model_max_length} tokens."""   )
													return input_ids












'''simple docstring'''







class 					__a      :

							def __init__(							self							:     List[Any]       ,     __magic_name__							:     int   )					->       None:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[Any]     						=  size
													UpperCAmelCase_			:      Tuple     						=  [0] * size
													UpperCAmelCase_			:      Optional[Any]     						=  [0] * size

							@staticmethod
							def     UpperCAmelCase__	(							__magic_name__							:     int   )					->       int:







													"""simple docstring"""
													return index | (index + 1)

							@staticmethod
							def     UpperCAmelCase__	(							__magic_name__							:     int   )					->       int:







													"""simple docstring"""
													return (index & (index + 1)) - 1

							def     UpperCAmelCase__	(							self							:     Optional[int]       ,     __magic_name__							:     int       ,     __magic_name__							:     int   )					->       None:







													"""simple docstring"""
													UpperCAmelCase_			:      int     						=  value
													while index < self.size:
																			UpperCAmelCase_			:      str     						=  self.get_prev(__magic_name__   ) + 1
																			if current_left_border == index:
																									UpperCAmelCase_			:      List[str]     						=  value
																			else:
																									UpperCAmelCase_			:      Optional[int]     						=  max(__magic_name__       ,     __magic_name__       ,     __magic_name__   )
																			UpperCAmelCase_			:      Tuple     						=  self.get_next(__magic_name__   )



							def     UpperCAmelCase__	(							self							:     Any       ,     __magic_name__							:     int       ,     __magic_name__							:     int   )					->       int:







													"""simple docstring"""
													right -= 1  # Because of right is exclusive
													UpperCAmelCase_			:      List[str]     						=  0
													while left <= right:
																			UpperCAmelCase_			:      Optional[Any]     						=  self.get_prev(__magic_name__   )
																			if left <= current_left:
																									UpperCAmelCase_			:      Dict     						=  max(__magic_name__       ,     self.tree[right]   )
																									UpperCAmelCase_			:      Optional[Any]     						=  current_left
																			else:
																									UpperCAmelCase_			:      str     						=  max(__magic_name__       ,     self.arr[right]   )
																									right -= 1
													return result


if __name__ == "__main__":
					import doctest

					doctest.testmod()





