

"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


A_							:			Dict											= logging.get_logger(__name__)

A_							:			Dict											= {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

A_							:			str											= {
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

A_							:			str											= {
    "distilbert-base-uncased": 512,
    "distilbert-base-uncased-distilled-squad": 512,
    "distilbert-base-cased": 512,
    "distilbert-base-cased-distilled-squad": 512,
    "distilbert-base-german-cased": 512,
    "distilbert-base-multilingual-cased": 512,
}


A_							:			Union[str, Any]											= {
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}





class    lowerCamelCase						(A__			):
			lowerCamelCase__   :		Optional[int]							=							VOCAB_FILES_NAMES
			lowerCamelCase__   :		Tuple							=							PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__   :		Dict							=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__   :		Tuple							=							PRETRAINED_INIT_CONFIGURATION
			lowerCamelCase__   :		Tuple							=							['input_ids', 'attention_mask']
			lowerCamelCase__   :		Optional[int]							=							DistilBertTokenizer

			def __init__(					self				:						str    ,				__UpperCAmelCase				:						Tuple=None    ,				__UpperCAmelCase				:						Dict=None    ,				__UpperCAmelCase				:						List[Any]=True    ,				__UpperCAmelCase				:						str="[UNK]"    ,				__UpperCAmelCase				:						Dict="[SEP]"    ,				__UpperCAmelCase				:						Optional[int]="[PAD]"    ,				__UpperCAmelCase				:						str="[CLS]"    ,				__UpperCAmelCase				:						Dict="[MASK]"    ,				__UpperCAmelCase				:						Optional[Any]=True    ,				__UpperCAmelCase				:						int=None    ,				**__UpperCAmelCase				:						Any    ,				)   ->    str:
						super().__init__(
						    __UpperCAmelCase    ,				tokenizer_file=__UpperCAmelCase    ,				do_lower_case=__UpperCAmelCase    ,				unk_token=__UpperCAmelCase    ,				sep_token=__UpperCAmelCase    ,				pad_token=__UpperCAmelCase    ,				cls_token=__UpperCAmelCase    ,				mask_token=__UpperCAmelCase    ,				tokenize_chinese_chars=__UpperCAmelCase    ,				strip_accents=__UpperCAmelCase    ,				**__UpperCAmelCase    ,				)

						SCREAMING_SNAKE_CASE__								=      json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
						if (
						    normalizer_state.get("""lowercase"""    ,				__UpperCAmelCase       ) != do_lower_case
						    or normalizer_state.get("""strip_accents"""    ,				__UpperCAmelCase       ) != strip_accents
						    or normalizer_state.get("""handle_chinese_chars"""    ,				__UpperCAmelCase       ) != tokenize_chinese_chars
						):
									SCREAMING_SNAKE_CASE__								=      getattr(__UpperCAmelCase    ,				normalizer_state.pop("""type"""       )       )
									SCREAMING_SNAKE_CASE__								=      do_lower_case
									SCREAMING_SNAKE_CASE__								=      strip_accents
									SCREAMING_SNAKE_CASE__								=      tokenize_chinese_chars
									SCREAMING_SNAKE_CASE__								=      normalizer_class(**__UpperCAmelCase       )

						SCREAMING_SNAKE_CASE__								=      do_lower_case

			def 					SCREAMING_SNAKE_CASE				(					self				:						int    ,				__UpperCAmelCase				:						Optional[Any]    ,				__UpperCAmelCase				:						int=None       )   ->    Tuple:
						SCREAMING_SNAKE_CASE__								=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

						return output

			def 					SCREAMING_SNAKE_CASE				(					self				:						Any    ,				__UpperCAmelCase				:						List[int]    ,				__UpperCAmelCase				:						Optional[List[int]] = None       )   ->    List[int]:
						SCREAMING_SNAKE_CASE__								=      [self.sep_token_id]
						SCREAMING_SNAKE_CASE__								=      [self.cls_token_id]
						if token_ids_a is None:
									return len(cls + token_ids_a + sep       ) * [0]
						return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]





			def 					SCREAMING_SNAKE_CASE				(					self				:						List[Any]    ,				__UpperCAmelCase				:						str    ,				__UpperCAmelCase				:						Optional[str] = None       )   ->    Tuple[str]:
						SCREAMING_SNAKE_CASE__								=      self._tokenizer.model.save(__UpperCAmelCase    ,				name=__UpperCAmelCase       )
						return tuple(__UpperCAmelCase       )





"""simple docstring"""




from .glue import GlueDataset, GlueDataTrainingArguments
from .language_modeling import (
    LineByLineTextDataset,
    LineByLineWithRefDataset,
    LineByLineWithSOPTextDataset,
    TextDataset,
    TextDatasetForNextSentencePrediction,
)
from .squad import SquadDataset, SquadDataTrainingArguments


