





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


_lowerCamelCase			       =      logging.get_logger(__name__)

_lowerCamelCase			       =      {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_lowerCamelCase			       =      {
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

_lowerCamelCase			       =      {
    'distilbert-base-uncased': 512,
    'distilbert-base-uncased-distilled-squad': 512,
    'distilbert-base-cased': 512,
    'distilbert-base-cased-distilled-squad': 512,
    'distilbert-base-german-cased': 512,
    'distilbert-base-multilingual-cased': 512,
}


_lowerCamelCase			       =      {
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}







class 			__A       (      lowerCamelCase__  ):




      """simple docstring"""







      UpperCAmelCase__						=      VOCAB_FILES_NAMES
      UpperCAmelCase__						=      PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase__						=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase__						=      PRETRAINED_INIT_CONFIGURATION
      UpperCAmelCase__						=      ["""input_ids""", """attention_mask"""]
      UpperCAmelCase__						=      DistilBertTokenizer


      def __init__(		self      ,							a__=None      ,							a__=None      ,							a__=True      ,							a__="[UNK]"      ,							a__="[SEP]"      ,							a__="[PAD]"      ,							a__="[CLS]"      ,							a__="[MASK]"      ,							a__=True      ,							a__=None      ,							**a__      ,							):



             """simple docstring"""





             super().__init__(
                 a__      ,							tokenizer_file=a__      ,							do_lower_case=a__      ,							unk_token=a__      ,							sep_token=a__      ,							pad_token=a__      ,							cls_token=a__      ,							mask_token=a__      ,							tokenize_chinese_chars=a__      ,							strip_accents=a__      ,							**a__      ,							)

             _lowerCamelCase							:     List[str]	=		json.loads(self.backend_tokenizer.normalizer.__getstate__())
             if (
                 normalizer_state.get('''lowercase'''      ,							a__) != do_lower_case
                 or normalizer_state.get('''strip_accents'''      ,							a__) != strip_accents
                 or normalizer_state.get('''handle_chinese_chars'''      ,							a__) != tokenize_chinese_chars
             ):
                    _lowerCamelCase							:     int	=		getattr(a__      ,							normalizer_state.pop('''type'''))
                    _lowerCamelCase							:     List[Any]	=		do_lower_case
                    _lowerCamelCase							:     str	=		strip_accents
                    _lowerCamelCase							:     str	=		tokenize_chinese_chars
                    _lowerCamelCase							:     Tuple	=		normalizer_class(**a__)

             _lowerCamelCase							:     int	=		do_lower_case


      def 				__snake_case			(		self      ,							a__      ,							a__=None):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                    output += token_ids_a + [self.sep_token_id]

             return output


      def 				__snake_case			(		self      ,							a__      ,							a__ = None):



             """simple docstring"""





             _lowerCamelCase							:     Tuple	=		[self.sep_token_id]
             _lowerCamelCase							:     Dict	=		[self.cls_token_id]
             if token_ids_a is None:
                    return len(cls + token_ids_a + sep) * [0]
             return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



      def 				__snake_case			(		self      ,							a__      ,							a__ = None):



             """simple docstring"""





             _lowerCamelCase							:     Optional[Any]	=		self._tokenizer.model.save(a__      ,							name=a__)
             return tuple(a__)








import importlib.util
import os
import platform
from argparse import ArgumentParser

import huggingface_hub

from .. import __version__ as version
from ..utils import (
    is_accelerate_available,
    is_flax_available,
    is_safetensors_available,
    is_tf_available,
    is_torch_available,
)
from . import BaseTransformersCLICommand




def    __UpperCAmelCase(      lowercase_      ):
       return EnvironmentCommand()




def    __UpperCAmelCase(      lowercase_      ):
       return EnvironmentCommand(args.accelerate_config_file      )







class 			__A       (      lowerCamelCase__  ):




      """simple docstring"""







      @staticmethod
      def 				__snake_case			(		a__):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		parser.add_parser('''env''')
             download_parser.set_defaults(func=a__)
             download_parser.add_argument(
                 '''--accelerate-config_file'''      ,							default=a__      ,							help='''The accelerate config file to use for the default values in the launching script.'''      ,							)
             download_parser.set_defaults(func=a__)


      def __init__(		self      ,							a__      ,							*a__):



             """simple docstring"""





             _lowerCamelCase							:     str	=		accelerate_config_file


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		'''not installed'''
             if is_safetensors_available():
                    import safetensors

                    _lowerCamelCase							:     Optional[Any]	=		safetensors.__version__
             elif importlib.util.find_spec('''safetensors''') is not None:
                    import safetensors

                    _lowerCamelCase							:     Optional[int]	=		F"""{safetensors.__version__} but is ignored because of PyTorch version too old."""

             _lowerCamelCase							:     Union[str, Any]	=		'''not installed'''
             _lowerCamelCase							:     Any	=		'''not found'''
             if is_accelerate_available():
                    import accelerate
                    from accelerate.commands.config import default_config_file, load_config_from_file

                    _lowerCamelCase							:     Optional[int]	=		accelerate.__version__
                    # Get the default from the config file.
                    if self._accelerate_config_file is not None or os.path.isfile(a__):
                           _lowerCamelCase							:     Optional[int]	=		load_config_from_file(self._accelerate_config_file).to_dict()

                    _lowerCamelCase							:     str	=		(
                        '''\n'''.join([F"""\t- {prop}: {val}""" for prop, val in accelerate_config.items()])
                        if isinstance(a__      ,							a__)
                        else F"""\t{accelerate_config}"""
                    )

             _lowerCamelCase							:     List[Any]	=		'''not installed'''
             _lowerCamelCase							:     Tuple	=		'''NA'''
             if is_torch_available():
                    import torch

                    _lowerCamelCase							:     int	=		torch.__version__
                    _lowerCamelCase							:     List[str]	=		torch.cuda.is_available()

             _lowerCamelCase							:     str	=		'''not installed'''
             _lowerCamelCase							:     Union[str, Any]	=		'''NA'''
             if is_tf_available():
                    import tensorflow as tf

                    _lowerCamelCase							:     List[str]	=		tf.__version__
                    try:
                           # deprecated in v2.1
                           _lowerCamelCase							:     Optional[int]	=		tf.test.is_gpu_available()
                    except AttributeError:
                           # returns list of devices, convert to bool
                           _lowerCamelCase							:     Optional[int]	=		bool(tf.config.list_physical_devices('''GPU'''))

             _lowerCamelCase							:     str	=		'''not installed'''
             _lowerCamelCase							:     List[Any]	=		'''not installed'''
             _lowerCamelCase							:     List[Any]	=		'''not installed'''
             _lowerCamelCase							:     Optional[int]	=		'''NA'''
             if is_flax_available():
                    import flax
                    import jax
                    import jaxlib

                    _lowerCamelCase							:     Any	=		flax.__version__
                    _lowerCamelCase							:     str	=		jax.__version__
                    _lowerCamelCase							:     Any	=		jaxlib.__version__
                    _lowerCamelCase							:     int	=		jax.lib.xla_bridge.get_backend().platform

             _lowerCamelCase							:     int	=		{
                 '''`transformers` version''': version,
                 '''Platform''': platform.platform(),
                 '''Python version''': platform.python_version(),
                 '''Huggingface_hub version''': huggingface_hub.__version__,
                 '''Safetensors version''': F"""{safetensors_version}""",
                 '''Accelerate version''': F"""{accelerate_version}""",
                 '''Accelerate config''': F"""{accelerate_config_str}""",
                 '''PyTorch version (GPU?)''': F"""{pt_version} ({pt_cuda_available})""",
                 '''Tensorflow version (GPU?)''': F"""{tf_version} ({tf_cuda_available})""",
                 '''Flax version (CPU?/GPU?/TPU?)''': F"""{flax_version} ({jax_backend})""",
                 '''Jax version''': F"""{jax_version}""",
                 '''JaxLib version''': F"""{jaxlib_version}""",
                 '''Using GPU in script?''': '''<fill in>''',
                 '''Using distributed or parallel set-up in script?''': '''<fill in>''',
             }

             print('''\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n''')
             print(self.format_dict(a__))

             return info



      @staticmethod
      def 				__snake_case			(		a__):



             """simple docstring"""





             return "\n".join([F"""- {prop}: {val}""" for prop, val in d.items()]) + "\n"

