



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


UpperCAmelCase_									=							logging.get_logger(__name__)

UpperCAmelCase_									=							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase_									=							{
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

UpperCAmelCase_									=							{
    "distilbert-base-uncased": 5_1_2,
    "distilbert-base-uncased-distilled-squad": 5_1_2,
    "distilbert-base-cased": 5_1_2,
    "distilbert-base-cased-distilled-squad": 5_1_2,
    "distilbert-base-german-cased": 5_1_2,
    "distilbert-base-multilingual-cased": 5_1_2,
}


UpperCAmelCase_									=							{
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}





class 		UpperCAmelCase						(						snake_case_				):
   SCREAMING_SNAKE_CASE__							=     VOCAB_FILES_NAMES
   SCREAMING_SNAKE_CASE__							=     PRETRAINED_VOCAB_FILES_MAP
   SCREAMING_SNAKE_CASE__							=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   SCREAMING_SNAKE_CASE__							=     PRETRAINED_INIT_CONFIGURATION
   SCREAMING_SNAKE_CASE__							=     ['''input_ids''', '''attention_mask''']
   SCREAMING_SNAKE_CASE__							=     DistilBertTokenizer





   def __init__(	self       ,     _lowerCAmelCase=None       ,     _lowerCAmelCase=None       ,     _lowerCAmelCase=True       ,     _lowerCAmelCase="[UNK]"       ,     _lowerCAmelCase="[SEP]"       ,     _lowerCAmelCase="[PAD]"       ,     _lowerCAmelCase="[CLS]"       ,     _lowerCAmelCase="[MASK]"       ,     _lowerCAmelCase=True       ,     _lowerCAmelCase=None       ,     **_lowerCAmelCase       ,     ):
         super().__init__(
             _lowerCAmelCase       ,     tokenizer_file=_lowerCAmelCase       ,     do_lower_case=_lowerCAmelCase       ,     unk_token=_lowerCAmelCase       ,     sep_token=_lowerCAmelCase       ,     pad_token=_lowerCAmelCase       ,     cls_token=_lowerCAmelCase       ,     mask_token=_lowerCAmelCase       ,     tokenize_chinese_chars=_lowerCAmelCase       ,     strip_accents=_lowerCAmelCase       ,     **_lowerCAmelCase       ,     )

         _lowerCAmelCase 			=  json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
         if (
             normalizer_state.get('''lowercase'''       ,     _lowerCAmelCase			) != do_lower_case
             or normalizer_state.get('''strip_accents'''       ,     _lowerCAmelCase			) != strip_accents
             or normalizer_state.get('''handle_chinese_chars'''       ,     _lowerCAmelCase			) != tokenize_chinese_chars
         ):
               _lowerCAmelCase 			=  getattr(_lowerCAmelCase       ,     normalizer_state.pop('''type'''			)			)
               _lowerCAmelCase 			=  do_lower_case
               _lowerCAmelCase 			=  strip_accents
               _lowerCAmelCase 			=  tokenize_chinese_chars
               _lowerCAmelCase 			=  normalizer_class(**_lowerCAmelCase			)

         _lowerCAmelCase 			=  do_lower_case





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase=None			):
         _lowerCAmelCase 			=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

         return output





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase = None			):
         _lowerCAmelCase 			=  [self.sep_token_id]
         _lowerCAmelCase 			=  [self.cls_token_id]
         if token_ids_a is None:
               return len(cls + token_ids_a + sep			) * [0]
         return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase = None			):
         _lowerCAmelCase 			=  self._tokenizer.model.save(_lowerCAmelCase       ,     name=_lowerCAmelCase			)
         return tuple(_lowerCAmelCase			)




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


UpperCAmelCase_									=							{"processing_layoutxlm": ["LayoutXLMProcessor"]}

try:
    if not is_sentencepiece_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    UpperCAmelCase_									=							["LayoutXLMTokenizer"]

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    UpperCAmelCase_									=							["LayoutXLMTokenizerFast"]

if TYPE_CHECKING:
    from .processing_layoutxlm import LayoutXLMProcessor

    try:
        if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_layoutxlm import LayoutXLMTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_layoutxlm_fast import LayoutXLMTokenizerFast

else:
    import sys

    UpperCAmelCase_									=							_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)