






'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


_A			:    Optional[int]					  =logging.get_logger(__name__)

_A			:    Optional[int]					  ={"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_A			:    Optional[int]					  ={
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

_A			:    Dict					  ={
    "distilbert-base-uncased": 512,
    "distilbert-base-uncased-distilled-squad": 512,
    "distilbert-base-cased": 512,
    "distilbert-base-cased-distilled-squad": 512,
    "distilbert-base-german-cased": 512,
    "distilbert-base-multilingual-cased": 512,
}


_A			:    Dict					  ={
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}

class _lowercase			(       UpperCamelCase_	):
							a          = VOCAB_FILES_NAMES
							a          = PRETRAINED_VOCAB_FILES_MAP
							a          = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							a          = PRETRAINED_INIT_CONFIGURATION
							a          = ["""input_ids""", """attention_mask"""]
							a          = DistilBertTokenizer




							def __init__(							self:       Optional[int]    ,							UpperCamelCase__:       List[str]=None    ,							UpperCamelCase__:       Optional[Any]=None    ,							UpperCamelCase__:       Any=True    ,							UpperCamelCase__:       Any="[UNK]"    ,							UpperCamelCase__:       Tuple="[SEP]"    ,							UpperCamelCase__:       Union[str, Any]="[PAD]"    ,							UpperCamelCase__:       Optional[int]="[CLS]"    ,							UpperCamelCase__:       Optional[int]="[MASK]"    ,							UpperCamelCase__:       Optional[Any]=True    ,							UpperCamelCase__:       Union[str, Any]=None    ,							**UpperCamelCase__:       int    ,							):
														super().__init__(
														    _a    ,							tokenizer_file=_a    ,							do_lower_case=_a    ,							unk_token=_a    ,							sep_token=_a    ,							pad_token=_a    ,							cls_token=_a    ,							mask_token=_a    ,							tokenize_chinese_chars=_a    ,							strip_accents=_a    ,							**_a    ,							)

														lowerCamelCase__      :			Union[str, Any] 						= json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
														if (
														    normalizer_state.get("""lowercase"""    ,							_a  ) != do_lower_case
														    or normalizer_state.get("""strip_accents"""    ,							_a  ) != strip_accents
														    or normalizer_state.get("""handle_chinese_chars"""    ,							_a  ) != tokenize_chinese_chars
														):
																					lowerCamelCase__      :			int 						= getattr(_a    ,							normalizer_state.pop("""type"""  )  )
																					lowerCamelCase__      :			Union[str, Any] 						= do_lower_case
																					lowerCamelCase__      :			Optional[Any] 						= strip_accents
																					lowerCamelCase__      :			Optional[Any] 						= tokenize_chinese_chars
																					lowerCamelCase__      :			Union[str, Any] 						= normalizer_class(**_a  )

														lowerCamelCase__      :			List[Any] 						= do_lower_case




							def 	lowerCamelCase_       (							self:       Dict    ,							UpperCamelCase__:       List[str]    ,							UpperCamelCase__:       Any=None  ):
														lowerCamelCase__      :			Optional[Any] 						= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output




							def 	lowerCamelCase_       (							self:       Optional[int]    ,							UpperCamelCase__:       List[str]    ,							UpperCamelCase__:       List[str] = None  ):
														lowerCamelCase__      :			List[str] 						= [self.sep_token_id]
														lowerCamelCase__      :			Tuple 						= [self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep  ) * [0]
														return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]




							def 	lowerCamelCase_       (							self:       str    ,							UpperCamelCase__:       List[Any]    ,							UpperCamelCase__:       Dict = None  ):
														lowerCamelCase__      :			str 						= self._tokenizer.model.save(_a    ,							name=_a  )
														return tuple(_a  )










'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_A			:    Dict					  ={
    '''configuration_git''': ['''GIT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''GitConfig''', '''GitVisionConfig'''],
    '''processing_git''': ['''GitProcessor'''],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A			:    Union[str, Any]					  =[
			    '''GIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''GitForCausalLM''',
			    '''GitModel''',
			    '''GitPreTrainedModel''',
			    '''GitVisionModel''',
			]

if TYPE_CHECKING:
			from .configuration_git import GIT_PRETRAINED_CONFIG_ARCHIVE_MAP, GitConfig, GitVisionConfig
			from .processing_git import GitProcessor

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_git import (
						    GIT_PRETRAINED_MODEL_ARCHIVE_LIST,
						    GitForCausalLM,
						    GitModel,
						    GitPreTrainedModel,
						    GitVisionModel,
						)

else:
			import sys

			_A			:    Dict					  =_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

