






'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


a_					= logging.get_logger(__name__)

a_					= {'vocab_file': 'vocab.txt'}

a_					= {
    'vocab_file': {
        'YituTech/conv-bert-base': 'https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt',
        'YituTech/conv-bert-medium-small': (
            'https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'
        ),
        'YituTech/conv-bert-small': 'https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt',
    }
}

a_					= {
    'YituTech/conv-bert-base': 5_1_2,
    'YituTech/conv-bert-medium-small': 5_1_2,
    'YituTech/conv-bert-small': 5_1_2,
}


a_					= {
    'YituTech/conv-bert-base': {'do_lower_case': True},
    'YituTech/conv-bert-medium-small': {'do_lower_case': True},
    'YituTech/conv-bert-small': {'do_lower_case': True},
}






class      __SCREAMING_SNAKE_CASE      (			lowerCamelCase				):
 snake_case_  							=							VOCAB_FILES_NAMES
 snake_case_  							=							PRETRAINED_VOCAB_FILES_MAP
 snake_case_  							=							PRETRAINED_INIT_CONFIGURATION
 snake_case_  							=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 snake_case_  							=							ConvBertTokenizer




 def __init__(      self	:							int					,      __lowercase	:							int=None					,      __lowercase	:							int=None					,      __lowercase	:							Dict=True					,      __lowercase	:							Optional[int]="[UNK]"					,      __lowercase	:							str="[SEP]"					,      __lowercase	:							Union[str, Any]="[PAD]"					,      __lowercase	:							str="[CLS]"					,      __lowercase	:							Dict="[MASK]"					,      __lowercase	:							List[str]=True					,      __lowercase	:							Dict=None					,      **__lowercase	:							Dict					,      )	->						List[Any]:
       super().__init__(
           __lowercase					,      tokenizer_file=__lowercase					,      do_lower_case=__lowercase					,      unk_token=__lowercase					,      sep_token=__lowercase					,      pad_token=__lowercase					,      cls_token=__lowercase					,      mask_token=__lowercase					,      tokenize_chinese_chars=__lowercase					,      strip_accents=__lowercase					,      **__lowercase					,      )

       SCREAMING_SNAKE_CASE__     :				Any									=json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
       if (
           normalizer_state.get('''lowercase'''					,      __lowercase      ) != do_lower_case
           or normalizer_state.get('''strip_accents'''					,      __lowercase      ) != strip_accents
           or normalizer_state.get('''handle_chinese_chars'''					,      __lowercase      ) != tokenize_chinese_chars
       ):
             SCREAMING_SNAKE_CASE__     :				int									=getattr(__lowercase					,      normalizer_state.pop('''type'''      )      )
             SCREAMING_SNAKE_CASE__     :				List[str]									=do_lower_case
             SCREAMING_SNAKE_CASE__     :				Dict									=strip_accents
             SCREAMING_SNAKE_CASE__     :				Any									=tokenize_chinese_chars
             SCREAMING_SNAKE_CASE__     :				List[str]									=normalizer_class(**__lowercase      )

       SCREAMING_SNAKE_CASE__     :				List[str]									=do_lower_case




 def      __magic_name__	(      self	:							int					,      __lowercase	:							Optional[Any]					,      __lowercase	:							Optional[int]=None      )	->						Union[str, Any]:
       SCREAMING_SNAKE_CASE__     :				List[Any]									=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

       return output




 def      __magic_name__	(      self	:							Union[str, Any]					,      __lowercase	:							List[int]					,      __lowercase	:							Optional[List[int]] = None      )	->						List[int]:
       SCREAMING_SNAKE_CASE__     :				Any									=[self.sep_token_id]
       SCREAMING_SNAKE_CASE__     :				Tuple									=[self.cls_token_id]
       if token_ids_a is None:
             return len(cls + token_ids_a + sep      ) * [0]
       return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]




 def      __magic_name__	(      self	:							int					,      __lowercase	:							str					,      __lowercase	:							Optional[str] = None      )	->						Tuple[str]:
       SCREAMING_SNAKE_CASE__     :				str									=self._tokenizer.model.save(__lowercase					,      name=__lowercase      )
       return tuple(__lowercase      )







'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_					= logging.get_logger(__name__)

a_					= {
    'bigcode/gpt_bigcode-santacoder': 'https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json',
}






class      __SCREAMING_SNAKE_CASE      (			lowerCamelCase				):
 snake_case_  							=							"""gpt_bigcode"""
 snake_case_  							=							["""past_key_values"""]
 snake_case_  							=							{
     """hidden_size""": """n_embd""",
     """max_position_embeddings""": """n_positions""",
     """num_attention_heads""": """n_head""",
     """num_hidden_layers""": """n_layer""",
 }




 def __init__(      self	:							Any					,      __lowercase	:							Any=5_02_57					,      __lowercase	:							int=10_24					,      __lowercase	:							List[str]=7_68					,      __lowercase	:							Optional[int]=12					,      __lowercase	:							Dict=12					,      __lowercase	:							List[str]=None					,      __lowercase	:							int="gelu_pytorch_tanh"					,      __lowercase	:							Union[str, Any]=0.1					,      __lowercase	:							Optional[int]=0.1					,      __lowercase	:							Optional[int]=0.1					,      __lowercase	:							Optional[Any]=1e-5					,      __lowercase	:							List[str]=0.02					,      __lowercase	:							Tuple=True					,      __lowercase	:							Optional[Any]=True					,      __lowercase	:							Union[str, Any]=5_02_56					,      __lowercase	:							List[Any]=5_02_56					,      __lowercase	:							Union[str, Any]=True					,      __lowercase	:							List[str]=True					,      __lowercase	:							Dict=True					,      **__lowercase	:							List[Any]					,      )	->						Optional[int]:
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=vocab_size
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=n_positions
       SCREAMING_SNAKE_CASE__     :				Dict									=n_embd
       SCREAMING_SNAKE_CASE__     :				Dict									=n_layer
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=n_head
       SCREAMING_SNAKE_CASE__     :				List[str]									=n_inner
       SCREAMING_SNAKE_CASE__     :				List[str]									=activation_function
       SCREAMING_SNAKE_CASE__     :				List[Any]									=resid_pdrop
       SCREAMING_SNAKE_CASE__     :				List[Any]									=embd_pdrop
       SCREAMING_SNAKE_CASE__     :				List[str]									=attn_pdrop
       SCREAMING_SNAKE_CASE__     :				Dict									=layer_norm_epsilon
       SCREAMING_SNAKE_CASE__     :				List[str]									=initializer_range
       SCREAMING_SNAKE_CASE__     :				List[Any]									=scale_attn_weights
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=use_cache
       SCREAMING_SNAKE_CASE__     :				Dict									=attention_softmax_in_fpaa
       SCREAMING_SNAKE_CASE__     :				int									=scale_attention_softmax_in_fpaa
       SCREAMING_SNAKE_CASE__     :				Dict									=multi_query

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=bos_token_id
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=eos_token_id

       super().__init__(bos_token_id=__lowercase					,      eos_token_id=__lowercase					,      **__lowercase      )