


'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


_lowerCAmelCase  :		Optional[int]		     =	logging.get_logger(__name__)

_lowerCAmelCase  :		Optional[int]		     =	{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_lowerCAmelCase  :		str		     =	{
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

_lowerCAmelCase  :		int		     =	{
    'distilbert-base-uncased': 512,
    'distilbert-base-uncased-distilled-squad': 512,
    'distilbert-base-cased': 512,
    'distilbert-base-cased-distilled-squad': 512,
    'distilbert-base-german-cased': 512,
    'distilbert-base-multilingual-cased': 512,
}


_lowerCAmelCase  :		Dict		     =	{
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}



class  lowerCAmelCase				(				a							):
	_lowerCamelCase   :					int			      = VOCAB_FILES_NAMES
	_lowerCamelCase   :					Tuple			      = PRETRAINED_VOCAB_FILES_MAP
	_lowerCamelCase   :					List[str]			      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowerCamelCase   :					Tuple			      = PRETRAINED_INIT_CONFIGURATION
	_lowerCamelCase   :					Tuple			      = ["""input_ids""", """attention_mask"""]
	_lowerCamelCase   :					List[Any]			      = DistilBertTokenizer



	def __init__(					self ,					snake_case__=None ,					snake_case__=None ,					snake_case__=True ,					snake_case__="[UNK]" ,					snake_case__="[SEP]" ,					snake_case__="[PAD]" ,					snake_case__="[CLS]" ,					snake_case__="[MASK]" ,					snake_case__=True ,					snake_case__=None ,					**snake_case__ ,					):
		super().__init__(
		    snake_case__ ,					tokenizer_file=snake_case__ ,					do_lower_case=snake_case__ ,					unk_token=snake_case__ ,					sep_token=snake_case__ ,					pad_token=snake_case__ ,					cls_token=snake_case__ ,					mask_token=snake_case__ ,					tokenize_chinese_chars=snake_case__ ,					strip_accents=snake_case__ ,					**snake_case__ ,					)

		lowerCAmelCase					:    Optional[int]      							= json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
		if (
		    normalizer_state.get('lowercase' ,					snake_case__    ) != do_lower_case
		    or normalizer_state.get('strip_accents' ,					snake_case__    ) != strip_accents
		    or normalizer_state.get('handle_chinese_chars' ,					snake_case__    ) != tokenize_chinese_chars
		):
			lowerCAmelCase					:    Any      							= getattr(snake_case__ ,					normalizer_state.pop('type'    )    )
			lowerCAmelCase					:    int      							= do_lower_case
			lowerCAmelCase					:    List[Any]      							= strip_accents
			lowerCAmelCase					:    List[Any]      							= tokenize_chinese_chars
			lowerCAmelCase					:    Optional[int]      							= normalizer_class(**snake_case__    )

		lowerCAmelCase					:    int      							= do_lower_case



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__=None    ):
		lowerCAmelCase					:    Optional[int]      							= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

		if token_ids_a:
			output += token_ids_a + [self.sep_token_id]

		return output



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ = None    ):
		lowerCAmelCase					:    int      							= [self.sep_token_id]
		lowerCAmelCase					:    Dict      							= [self.cls_token_id]
		if token_ids_a is None:
			return len(cls + token_ids_a + sep    ) * [0]
		return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ = None    ):
		lowerCAmelCase					:    int      							= self._tokenizer.model.save(snake_case__ ,					name=snake_case__    )
		return tuple(snake_case__    )




'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


_lowerCAmelCase  :		int		     =	{
    'configuration_nezha': ['NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'NezhaConfig'],
}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_lowerCAmelCase  :		Tuple		     =	[
				    'NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'NezhaForNextSentencePrediction',
				    'NezhaForMaskedLM',
				    'NezhaForPreTraining',
				    'NezhaForMultipleChoice',
				    'NezhaForQuestionAnswering',
				    'NezhaForSequenceClassification',
				    'NezhaForTokenClassification',
				    'NezhaModel',
				    'NezhaPreTrainedModel',
				]


if TYPE_CHECKING:
				from .configuration_nezha import NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP, NezhaConfig

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_nezha import (
								    NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST,
								    NezhaForMaskedLM,
								    NezhaForMultipleChoice,
								    NezhaForNextSentencePrediction,
								    NezhaForPreTraining,
								    NezhaForQuestionAnswering,
								    NezhaForSequenceClassification,
								    NezhaForTokenClassification,
								    NezhaModel,
								    NezhaPreTrainedModel,
								)


else:
				import sys

				_lowerCAmelCase  :		str		     =	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
