






from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowerCamelCase       :	List[Any]									=     logging.get_logger(__name__)


lowerCamelCase       :	Any									=     {
    "Salesforce/codegen-350M-nl": "https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json",
    "Salesforce/codegen-350M-multi": "https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json",
    "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json",
    "Salesforce/codegen-2B-nl": "https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json",
    "Salesforce/codegen-2B-multi": "https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json",
    "Salesforce/codegen-2B-mono": "https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json",
    "Salesforce/codegen-6B-nl": "https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json",
    "Salesforce/codegen-6B-multi": "https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json",
    "Salesforce/codegen-6B-mono": "https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json",
    "Salesforce/codegen-16B-nl": "https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json",
    "Salesforce/codegen-16B-multi": "https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json",
    "Salesforce/codegen-16B-mono": "https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json",
}







class 						A(		UpperCamelCase     ):



 '''simple docstring'''


 UpperCamelCase		     =		'''codegen'''
 UpperCamelCase		     =		{
     '''max_position_embeddings''': '''n_positions''',
     '''hidden_size''': '''n_embd''',
     '''num_attention_heads''': '''n_head''',
     '''num_hidden_layers''': '''n_layer''',
 }

 def __init__(       self      :						Optional[Any]		,							A_      :						List[str]=50400		,							A_      :						str=2048		,							A_      :						int=2048		,							A_      :						Union[str, Any]=4096		,							A_      :						str=28		,							A_      :						Optional[Any]=16		,							A_      :						Dict=64		,							A_      :						int=None		,							A_      :						Optional[Any]="gelu_new"		,							A_      :						List[Any]=0.0		,							A_      :						Dict=0.0		,							A_      :						Dict=0.0		,							A_      :						Dict=1E-5		,							A_      :						Union[str, Any]=0.02		,							A_      :						Optional[Any]=True		,							A_      :						Union[str, Any]=50256		,							A_      :						Optional[int]=50256		,							A_      :						Dict=False		,							**A_      :						Union[str, Any]		,							)     ->			List[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					vocab_size
       lowerCamelCase_	  =					n_ctx
       lowerCamelCase_	  =					n_positions
       lowerCamelCase_	  =					n_embd
       lowerCamelCase_	  =					n_layer
       lowerCamelCase_	  =					n_head
       lowerCamelCase_	  =					n_inner
       lowerCamelCase_	  =					rotary_dim
       lowerCamelCase_	  =					activation_function
       lowerCamelCase_	  =					resid_pdrop
       lowerCamelCase_	  =					embd_pdrop
       lowerCamelCase_	  =					attn_pdrop
       lowerCamelCase_	  =					layer_norm_epsilon
       lowerCamelCase_	  =					initializer_range
       lowerCamelCase_	  =					use_cache

       lowerCamelCase_	  =					bos_token_id
       lowerCamelCase_	  =					eos_token_id

       super().__init__(
           bos_token_id=A_		,							eos_token_id=A_		,							tie_word_embeddings=A_		,							**A_						)







class 						A(		UpperCamelCase     ):



 '''simple docstring'''


 def __init__(       self      :						int		,							A_      :						PretrainedConfig		,							A_      :						str = "default"		,							A_      :						List[PatchingSpec] = None		,							A_      :						bool = False		,							)     ->			Optional[Any]:

       """simple docstring"""






       super().__init__(A_		,							task=A_		,							patching_specs=A_		,							use_past=A_						)
       if not getattr(self._config		,							'pad_token_id'		,							A_						):
             # TODO: how to do that better?
             lowerCamelCase_	  =					0

 @property
 def        a__				(       self      :						Union[str, Any]						)     ->			Mapping[str, Mapping[int, str]]:

       """simple docstring"""






       lowerCamelCase_	  =					OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}						)
       if self.use_past:
             self.fill_with_past_key_values_(A_		,							direction='inputs'						)
             lowerCamelCase_	  =					{0: 'batch', 1: 'past_sequence + sequence'}
       else:
             lowerCamelCase_	  =					{0: 'batch', 1: 'sequence'}

       return common_inputs

 @property
 def        a__				(       self      :						str						)     ->			int:

       """simple docstring"""






       return self._config.n_layer

 @property
 def        a__				(       self      :						List[str]						)     ->			int:

       """simple docstring"""






       return self._config.n_head

 def        a__				(       self      :						Union[str, Any]		,							A_      :						PreTrainedTokenizer		,							A_      :						int = -1		,							A_      :						int = -1		,							A_      :						bool = False		,							A_      :						Optional[TensorType] = None		,							)     ->			Mapping[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					super(A_		,							self						).generate_dummy_inputs(
           A_		,							batch_size=A_		,							seq_length=A_		,							is_pair=A_		,							framework=A_						)

       # We need to order the input in the way they appears in the forward()
       lowerCamelCase_	  =					OrderedDict({'input_ids': common_inputs['input_ids']}						)

       # Need to add the past_keys
       if self.use_past:
             if not is_torch_available():
                   raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'						)
             else:
                   import torch

                   lowerCamelCase_  ,       lowerCamelCase_	  =					common_inputs['input_ids'].shape
                   # Not using the same length for past_key_values
                   lowerCamelCase_	  =					seqlen + 2
                   lowerCamelCase_	  =					(
                       batch,
                       self.num_attention_heads,
                       past_key_values_length,
                       self._config.hidden_size // self.num_attention_heads,
                   )
                   lowerCamelCase_	  =					[
                       (torch.zeros(A_						), torch.zeros(A_						)) for _ in range(self.num_layers						)
                   ]

       lowerCamelCase_	  =					common_inputs['attention_mask']
       if self.use_past:
             lowerCamelCase_	  =					ordered_inputs['attention_mask'].dtype
             lowerCamelCase_	  =					torch.cat(
                 [ordered_inputs['attention_mask'], torch.ones(A_		,							A_		,							dtype=A_						)]		,							dim=1						)

       return ordered_inputs





 @property
 def        a__				(       self      :						List[Any]						)     ->			int:

       """simple docstring"""






       return 13









import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation

lowerCamelCase       :	str									=     logging.get_logger(__name__)


lowerCamelCase       :	Optional[Any]									=     {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

lowerCamelCase       :	int									=     {
    "vocab_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"},
    "merges_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"},
    "tokenizer_config_file": {
        "facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"
    },
}

lowerCamelCase       :	Tuple									=     {"facebook/blenderbot-3B": 128}

@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def     _SCREAMING_SNAKE_CASE    (							):


      '''simple docstring'''



      lowerCamelCase_	  =					(
          list(range(ord('!'  )			,							ord('~'  ) + 1  )  ) + list(range(ord('¡'  )			,							ord('¬'  ) + 1  )  ) + list(range(ord('®'  )			,							ord('ÿ'  ) + 1  )  )
      )
      lowerCamelCase_	  =					bs[:]
      lowerCamelCase_	  =					0
      for b in range(2**8  ):
            if b not in bs:
                  bs.append(lowercase  )
                  cs.append(2**8 + n  )
                  n += 1
      lowerCamelCase_	  =					[chr(lowercase  ) for n in cs]
      return dict(zip(lowercase			,							lowercase  )  )

def     _SCREAMING_SNAKE_CASE    (							lowercase			: int  ):


      '''simple docstring'''



      lowerCamelCase_	  =					set()
      lowerCamelCase_	  =					word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)  )
            lowerCamelCase_	  =					char
      return pairs







class 						A(		UpperCamelCase     ):



 '''simple docstring'''


 UpperCamelCase		     =		VOCAB_FILES_NAMES
 UpperCamelCase		     =		PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase		     =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCamelCase		     =		['''input_ids''', '''attention_mask''']

 def __init__(       self      :						Optional[Any]		,							A_      :						List[Any]		,							A_      :						List[Any]		,							A_      :						Union[str, Any]="replace"		,							A_      :						Dict="<s>"		,							A_      :						Optional[int]="</s>"		,							A_      :						Optional[Any]="</s>"		,							A_      :						Dict="<s>"		,							A_      :						Dict="<unk>"		,							A_      :						Any="<pad>"		,							A_      :						Dict="<mask>"		,							A_      :						Union[str, Any]=False		,							**A_      :						List[str]		,							)     ->			Tuple:

       """simple docstring"""






       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else bos_token
       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else eos_token
       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else sep_token
       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else cls_token
       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else unk_token
       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       lowerCamelCase_	  =					AddedToken(A_		,							lstrip=A_		,							rstrip=A_						) if isinstance(A_		,							A_						) else mask_token

       super().__init__(
           errors=A_		,							bos_token=A_		,							eos_token=A_		,							unk_token=A_		,							sep_token=A_		,							cls_token=A_		,							pad_token=A_		,							mask_token=A_		,							add_prefix_space=A_		,							**A_		,							)

       with open(A_		,							encoding='utf-8'						) as vocab_handle:
             lowerCamelCase_	  =					json.load(A_						)
       lowerCamelCase_	  =					{v: k for k, v in self.encoder.items()}
       lowerCamelCase_	  =					errors  # how to handle errors in decoding
       lowerCamelCase_	  =					bytes_to_unicode()
       lowerCamelCase_	  =					{v: k for k, v in self.byte_encoder.items()}
       with open(A_		,							encoding='utf-8'						) as merges_handle:
             lowerCamelCase_	  =					merges_handle.read().split('\n'						)[1:-1]
       lowerCamelCase_	  =					[tuple(merge.split()						) for merge in bpe_merges]
       lowerCamelCase_	  =					dict(zip(A_		,							range(len(A_						)						)						)						)
       lowerCamelCase_	  =					{}
       lowerCamelCase_	  =					add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       lowerCamelCase_	  =					re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'						)

 @property
 # Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
 def        a__				(       self      :						Optional[Any]						)     ->			Dict:

       """simple docstring"""






       return len(self.encoder						)

 def        a__				(       self      :						List[Any]						)     ->			Dict:

       """simple docstring"""






       return dict(self.encoder		,							**self.added_tokens_encoder						)

 def        a__				(       self      :						Tuple		,							A_      :						Tuple						)     ->			Optional[Any]:

       """simple docstring"""






       if token in self.cache:
             return self.cache[token]
       lowerCamelCase_	  =					tuple(A_						)
       lowerCamelCase_	  =					get_pairs(A_						)

       if not pairs:
             return token

       while True:
             lowerCamelCase_	  =					min(A_		,							key=lambda A_						: self.bpe_ranks.get(A_		,							float('inf'						)						)						)
             if bigram not in self.bpe_ranks:
                   break
             lowerCamelCase_  ,       lowerCamelCase_	  =					bigram
             lowerCamelCase_	  =					[]
             lowerCamelCase_	  =					0
             while i < len(A_						):
                   try:
                         lowerCamelCase_	  =					word.index(A_		,							A_						)
                   except ValueError:
                         new_word.extend(word[i:]						)
                         break
                   else:
                         new_word.extend(word[i:j]						)
                         lowerCamelCase_	  =					j

                   if word[i] == first and i < len(A_						) - 1 and word[i + 1] == second:
                         new_word.append(first + second						)
                         i += 2
                   else:
                         new_word.append(word[i]						)
                         i += 1
             lowerCamelCase_	  =					tuple(A_						)
             lowerCamelCase_	  =					new_word
             if len(A_						) == 1:
                   break
             else:
                   lowerCamelCase_	  =					get_pairs(A_						)
       lowerCamelCase_	  =					' '.join(A_						)
       lowerCamelCase_	  =					word
       return word

 def        a__				(       self      :						str		,							A_      :						List[str]						)     ->			List[str]:

       """simple docstring"""






       lowerCamelCase_	  =					[]
       for token in re.findall(self.pat		,							A_						):
             lowerCamelCase_	  =					''.join(
                 self.byte_encoder[b] for b in token.encode('utf-8'						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
             bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A_						).split(' '						)						)
       return bpe_tokens

 def        a__				(       self      :						Tuple		,							A_      :						str						)     ->			Optional[Any]:

       """simple docstring"""






       return self.encoder.get(A_		,							self.encoder.get(self.unk_token						)						)

 def        a__				(       self      :						Tuple		,							A_      :						Dict						)     ->			List[Any]:

       """simple docstring"""






       return self.decoder.get(A_						)

 def        a__				(       self      :						Optional[int]		,							A_      :						List[Any]						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					''.join(A_						)
       lowerCamelCase_	  =					bytearray([self.byte_decoder[c] for c in text]						).decode('utf-8'		,							errors=self.errors						)
       return text

 def        a__				(       self      :						Tuple		,							A_      :						str		,							A_      :						Optional[str] = None						)     ->			Tuple[str]:

       """simple docstring"""






       if not os.path.isdir(A_						):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
             return
       lowerCamelCase_	  =					os.path.join(
           A_		,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)
       lowerCamelCase_	  =					os.path.join(
           A_		,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']						)

       with open(A_		,							'w'		,							encoding='utf-8'						) as f:
             f.write(json.dumps(self.encoder		,							indent=2		,							sort_keys=A_		,							ensure_ascii=A_						) + '\n'						)

       lowerCamelCase_	  =					0
       with open(A_		,							'w'		,							encoding='utf-8'						) as writer:
             writer.write('#version: 0.2\n'						)
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,							key=lambda A_						: kv[1]						):
                   if index != token_index:
                         logger.warning(
                             f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                             ' Please check that the tokenizer is not corrupted!'						)
                         lowerCamelCase_	  =					token_index
                   writer.write(' '.join(A_						) + '\n'						)
                   index += 1

       return vocab_file, merge_file

 def        a__				(       self      :						str		,							A_      :						List[int]		,							A_      :						Optional[List[int]] = None		,							A_      :						bool = False						)     ->			List[int]:

       """simple docstring"""






       if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=A_		,							token_ids_a=A_		,							already_has_special_tokens=A_						)

       if token_ids_a is None:
             return [1] + ([0] * len(A_						)) + [1]
       return [1] + ([0] * len(A_						)) + [1, 1] + ([0] * len(A_						)) + [1]

 def        a__				(       self      :						int		,							A_      :						List[int]		,							A_      :						Optional[List[int]] = None						)     ->			List[int]:

       """simple docstring"""






       lowerCamelCase_	  =					[self.sep_token_id]
       lowerCamelCase_	  =					[self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]

 def        a__				(       self      :						str		,							A_      :						Optional[Any]		,							A_      :						Union[str, Any]=False		,							**A_      :						List[str]						)     ->			List[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					kwargs.pop('add_prefix_space'		,							self.add_prefix_space						)
       if (is_split_into_words or add_prefix_space) and (len(A_						) > 0 and not text[0].isspace()):
             lowerCamelCase_	  =					' ' + text
       return (text, kwargs)

 def        a__				(       self      :						List[Any]		,							A_      :						List[int]		,							A_      :						Optional[List[int]] = None						)     ->			Dict:

       """simple docstring"""






       return token_ids_a + [self.eos_token_id]





 def        a__				(       self      :						Optional[int]		,							A_      :						"Conversation"						)     ->			List[int]:

       """simple docstring"""






       lowerCamelCase_	  =					[]
       for is_user, text in conversation.iter_texts():
             if is_user:
                   # We need to space prefix as it's being done within blenderbot
                   inputs.append(' ' + text						)
             else:
                   # Generated responses should contain them already.
                   inputs.append(A_						)

       lowerCamelCase_	  =					'  '.join(A_						)
       lowerCamelCase_	  =					self.encode(A_						)
       if len(A_						) > self.model_max_length:
             lowerCamelCase_	  =					input_ids[-self.model_max_length :]
             logger.warning(f"""Trimmed input from conversation as it was longer than {self.model_max_length} tokens."""						)
       return input_ids



