


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


lowerCAmelCase							=						logging.get_logger(__name__)

lowerCAmelCase							=						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase							=						{
    """vocab_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-german-cased""": """https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt""",
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-german-cased""": (
            """https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase							=						{
    """distilbert-base-uncased""": 512,
    """distilbert-base-uncased-distilled-squad""": 512,
    """distilbert-base-cased""": 512,
    """distilbert-base-cased-distilled-squad""": 512,
    """distilbert-base-german-cased""": 512,
    """distilbert-base-multilingual-cased""": 512,
}


lowerCAmelCase							=						{
    """distilbert-base-uncased""": {"""do_lower_case""": True},
    """distilbert-base-uncased-distilled-squad""": {"""do_lower_case""": True},
    """distilbert-base-cased""": {"""do_lower_case""": False},
    """distilbert-base-cased-distilled-squad""": {"""do_lower_case""": False},
    """distilbert-base-german-cased""": {"""do_lower_case""": False},
    """distilbert-base-multilingual-cased""": {"""do_lower_case""": False},
}




class        lowerCamelCase      (     _UpperCamelCase   ):
	_lowerCAmelCase    :		Any        =	VOCAB_FILES_NAMES
	_lowerCAmelCase    :		Tuple        =	PRETRAINED_VOCAB_FILES_MAP
	_lowerCAmelCase    :		str        =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowerCAmelCase    :		List[Any]        =	PRETRAINED_INIT_CONFIGURATION
	_lowerCAmelCase    :		Union[str, Any]        =	['''input_ids''', '''attention_mask''']
	_lowerCAmelCase    :		Dict        =	DistilBertTokenizer


	def __init__(						self							,    lowercase__=None							,    lowercase__=None							,    lowercase__=True							,    lowercase__="[UNK]"							,    lowercase__="[SEP]"							,    lowercase__="[PAD]"							,    lowercase__="[CLS]"							,    lowercase__="[MASK]"							,    lowercase__=True							,    lowercase__=None							,    **lowercase__							,    ):
							super().__init__(
							    lowercase__							,    tokenizer_file=lowercase__							,    do_lower_case=lowercase__							,    unk_token=lowercase__							,    sep_token=lowercase__							,    pad_token=lowercase__							,    cls_token=lowercase__							,    mask_token=lowercase__							,    tokenize_chinese_chars=lowercase__							,    strip_accents=lowercase__							,    **lowercase__							,    )

							__UpperCAmelCase :       int  						=  json.loads(self.backend_tokenizer.normalizer.__getstate__())
							if (
							    normalizer_state.get('''lowercase'''							,    lowercase__) != do_lower_case
							    or normalizer_state.get('''strip_accents'''							,    lowercase__) != strip_accents
							    or normalizer_state.get('''handle_chinese_chars'''							,    lowercase__) != tokenize_chinese_chars
							):
													__UpperCAmelCase :       Any  						=  getattr(lowercase__							,    normalizer_state.pop('''type'''))
													__UpperCAmelCase :       Dict  						=  do_lower_case
													__UpperCAmelCase :       Optional[Any]  						=  strip_accents
													__UpperCAmelCase :       str  						=  tokenize_chinese_chars
													__UpperCAmelCase :       Tuple  						=  normalizer_class(**lowercase__)

							__UpperCAmelCase :       str  						=  do_lower_case


	def    A(						self							,    lowercase__							,    lowercase__=None):
							__UpperCAmelCase :       Optional[int]  						=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

							return output


	def    A(						self							,    lowercase__							,    lowercase__ = None):
							__UpperCAmelCase :       Tuple  						=  [self.sep_token_id]
							__UpperCAmelCase :       Union[str, Any]  						=  [self.cls_token_id]
							if token_ids_a is None:
													return len(cls + token_ids_a + sep) * [0]
							return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


	def    A(						self							,    lowercase__							,    lowercase__ = None):
							__UpperCAmelCase :       int  						=  self._tokenizer.model.save(lowercase__							,    name=lowercase__)
							return tuple(lowercase__)





def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			)		->    str:




						'''simple docstring'''







						if a < 0 or b < 0:
												raise ValueError('''the value of both inputs must be positive'''			)
						__UpperCAmelCase :       Dict  						=  str(bin(lowercase_			)			)[2:]  # remove the leading "0b"
						__UpperCAmelCase :       List[Any]  						=  str(bin(lowercase_			)			)[2:]
						__UpperCAmelCase :       List[Any]  						=  max(len(lowercase_			)			,					len(lowercase_			)			)
						return "0b" + "".join(
						    str(int('''1''' in (char_a, char_b)			)			)
						    for char_a, char_b in zip(a_binary.zfill(lowercase_			)			,					b_binary.zfill(lowercase_			)			)			)


if __name__ == "__main__":
				import doctest

				doctest.testmod()

