"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


_UpperCamelCase     :		Union[str, Any]											=					logging.get_logger(__name__)

_UpperCamelCase     :		Dict											=					{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_UpperCamelCase     :		List[str]											=					{
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

_UpperCamelCase     :		Tuple											=					{
    "distilbert-base-uncased": 5_12,
    "distilbert-base-uncased-distilled-squad": 5_12,
    "distilbert-base-cased": 5_12,
    "distilbert-base-cased-distilled-squad": 5_12,
    "distilbert-base-german-cased": 5_12,
    "distilbert-base-multilingual-cased": 5_12,
}


_UpperCamelCase     :		List[Any]											=					{
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}


class   UpperCAmelCase_			(					_UpperCamelCase):
						lowerCamelCase__	:					Tuple										=						VOCAB_FILES_NAMES
						lowerCamelCase__	:					Optional[int]										=						PRETRAINED_VOCAB_FILES_MAP
						lowerCamelCase__	:					Union[str, Any]										=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCamelCase__	:					Dict										=						PRETRAINED_INIT_CONFIGURATION
						lowerCamelCase__	:					Optional[Any]										=						["input_ids", "attention_mask"]
						lowerCamelCase__	:					Union[str, Any]										=						DistilBertTokenizer


						def __init__(     self	,    a=None	,    a=None	,    a=True	,    a="[UNK]"	,    a="[SEP]"	,    a="[PAD]"	,    a="[CLS]"	,    a="[MASK]"	,    a=True	,    a=None	,    **a	,    ) ->      Optional[Any]:
													super().__init__(
													    a	,    tokenizer_file=a	,    do_lower_case=a	,    unk_token=a	,    sep_token=a	,    pad_token=a	,    cls_token=a	,    mask_token=a	,    tokenize_chinese_chars=a	,    strip_accents=a	,    **a	,    )

													lowercase__    :	Dict     					=			json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
													if (
													    normalizer_state.get('lowercase'	,    a							) != do_lower_case
													    or normalizer_state.get('strip_accents'	,    a							) != strip_accents
													    or normalizer_state.get('handle_chinese_chars'	,    a							) != tokenize_chinese_chars
													):
																				lowercase__    :	List[Any]     					=			getattr(a	,    normalizer_state.pop('type'							)							)
																				lowercase__    :	Optional[int]     					=			do_lower_case
																				lowercase__    :	Any     					=			strip_accents
																				lowercase__    :	int     					=			tokenize_chinese_chars
																				lowercase__    :	Dict     					=			normalizer_class(**a							)

													lowercase__    :	List[Any]     					=			do_lower_case


						def 	_UpperCAmelCase (     self	,    a	,    a=None							) ->      Union[str, Any]:
													lowercase__    :	Optional[int]     					=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

													if token_ids_a:
																				output += token_ids_a + [self.sep_token_id]

													return output


						def 	_UpperCAmelCase (     self	,    a	,    a = None							) ->      Optional[Any]:
													lowercase__    :	Optional[Any]     					=			[self.sep_token_id]
													lowercase__    :	str     					=			[self.cls_token_id]
													if token_ids_a is None:
																				return len(cls + token_ids_a + sep							) * [0]
													return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]


						def 	_UpperCAmelCase (     self	,    a	,    a = None							) ->      List[Any]:
													lowercase__    :	List[str]     					=			self._tokenizer.model.save(a	,    name=a							)
													return tuple(a							)


"""simple docstring"""





# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
       import torch


_UpperCamelCase     :		int											=					logging.get_logger(__name__)


@dataclass
class   UpperCAmelCase_			:


      def __init__(     self	,    a=False	,    a=False	,    a=6.0	,    a=None	,    a=False	,    a=False	,    a=None	,    a="fp4"	,    a=False	,    **a	,    ) ->      Tuple:
             lowercase__    :	str     					=			load_in_abit
             lowercase__    :	str     					=			load_in_abit
             lowercase__    :	List[str]     					=			llm_inta_threshold
             lowercase__    :	Dict     					=			llm_inta_skip_modules
             lowercase__    :	Tuple     					=			llm_inta_enable_fpaa_cpu_offload
             lowercase__    :	Any     					=			llm_inta_has_fpaa_weight
             lowercase__    :	Any     					=			bnb_abit_quant_type
             lowercase__    :	Dict     					=			bnb_abit_use_double_quant

             if bnb_abit_compute_dtype is None:
                    lowercase__    :	Dict     					=			torch.floataa
             elif isinstance(a	,    a							):
                    lowercase__    :	Any     					=			getattr(a	,    a							)
             elif isinstance(a	,    torch.dtype							):
                    lowercase__    :	Any     					=			bnb_abit_compute_dtype
             else:
                    raise ValueError('bnb_4bit_compute_dtype must be a string or a torch.dtype'							)

             self.post_init()


      def 	_UpperCAmelCase (     self							) ->      str:
             if not isinstance(self.llm_inta_threshold	,    a							):
                    raise ValueError('llm_int8_threshold must be a float'							)

             if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules	,    a							):
                    raise ValueError('llm_int8_skip_modules must be a list of strings'							)
             if not isinstance(self.llm_inta_enable_fpaa_cpu_offload	,    a							):
                    raise ValueError('llm_int8_enable_fp32_cpu_offload must be a boolean'							)

             if not isinstance(self.llm_inta_has_fpaa_weight	,    a							):
                    raise ValueError('llm_int8_has_fp16_weight must be a boolean'							)

             if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype	,    torch.dtype							):
                    raise ValueError('bnb_4bit_compute_dtype must be torch.dtype'							)

             if not isinstance(self.bnb_abit_quant_type	,    a							):
                    raise ValueError('bnb_4bit_quant_type must be a string'							)

             if not isinstance(self.bnb_abit_use_double_quant	,    a							):
                    raise ValueError('bnb_4bit_use_double_quant must be a boolean'							)

             if self.load_in_abit and not version.parse(importlib.metadata.version('bitsandbytes'							)							) >= version.parse(
                 '0.39.0'							):
                    raise ValueError(
                        '4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version'							)


      def 	_UpperCAmelCase (     self							) ->      Tuple:
             return self.load_in_abit or self.load_in_abit


      def 	_UpperCAmelCase (     self							) ->      List[str]:
             if self.load_in_abit:
                    return "llm_int8"
             elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
                    return "fp4"
             elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
                    return "nf4"
             else:
                    return None


      @classmethod
      def 	_UpperCAmelCase (     cls	,    a	,    a	,    **a							) ->      Optional[Any]:
             lowercase__    :	List[Any]     					=			cls(**a							)

             lowercase__    :	Union[str, Any]     					=			[]
             for key, value in kwargs.items():
                    if hasattr(a	,    a							):
                           setattr(a	,    a	,    a							)
                           to_remove.append(a							)
             for key in to_remove:
                    kwargs.pop(a	,    a							)

             if return_unused_kwargs:
                    return config, kwargs
             else:
                    return config


      def 	_UpperCAmelCase (     self	,    a							) ->      Dict:
             with open(a	,    'w'	,    encoding='utf-8'							) as writer:
                    lowercase__    :	Any     					=			self.to_dict()
                    lowercase__    :	str     					=			json.dumps(a	,    indent=2	,    sort_keys=a							) + '\n'

                    writer.write(a							)


      def 	_UpperCAmelCase (     self							) ->      Dict[str, Any]:
             lowercase__    :	Optional[Any]     					=			copy.deepcopy(self.__dict__							)
             lowercase__    :	Any     					=			str(output['bnb_4bit_compute_dtype']							).split('.'							)[1]

             return output


      def __repr__(     self							) ->      Dict:
             return f"""{self.__class__.__name__} {self.to_json_string()}"""


      def 	_UpperCAmelCase (     self	,    a = True							) ->      str:
             if use_diff is True:
                    lowercase__    :	List[Any]     					=			self.to_diff_dict()
             else:
                    lowercase__    :	List[str]     					=			self.to_dict()
             return json.dumps(a	,    indent=2	,    sort_keys=a							) + "\n"


      def 	_UpperCAmelCase (     self							) ->      Dict[str, Any]:
             lowercase__    :	Tuple     					=			self.to_dict()

             # get the default config dict
             lowercase__    :	Optional[Any]     					=			BitsAndBytesConfig().to_dict()

             lowercase__    :	int     					=			{}

             # only serialize values that differ from the default config
             for key, value in config_dict.items():
                    if value != default_config_dict[key]:
                           lowercase__    :	Optional[int]     					=			value

             return serializable_config_dict

