"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


UpperCAmelCase_:					str     =					logging.get_logger(__name__)

UpperCAmelCase_:					Dict     =					{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase_:					int     =					{
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

UpperCAmelCase_:					Any     =					{
    "distilbert-base-uncased": 5_1_2,
    "distilbert-base-uncased-distilled-squad": 5_1_2,
    "distilbert-base-cased": 5_1_2,
    "distilbert-base-cased-distilled-squad": 5_1_2,
    "distilbert-base-german-cased": 5_1_2,
    "distilbert-base-multilingual-cased": 5_1_2,
}


UpperCAmelCase_:					Optional[Any]     =					{
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}






class    a__    (   __lowerCAmelCase	):
   snake_case_							=      VOCAB_FILES_NAMES
   snake_case_							=      PRETRAINED_VOCAB_FILES_MAP
   snake_case_							=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   snake_case_							=      PRETRAINED_INIT_CONFIGURATION
   snake_case_							=      ["input_ids", "attention_mask"]
   snake_case_							=      DistilBertTokenizer


   def __init__(     self,				_UpperCAmelCase=None,				_UpperCAmelCase=None,				_UpperCAmelCase=True,				_UpperCAmelCase="[UNK]",				_UpperCAmelCase="[SEP]",				_UpperCAmelCase="[PAD]",				_UpperCAmelCase="[CLS]",				_UpperCAmelCase="[MASK]",				_UpperCAmelCase=True,				_UpperCAmelCase=None,				**_UpperCAmelCase,				):
        '''simple docstring'''



        super().__init__(
            _UpperCamelCase,				tokenizer_file=_UpperCamelCase,				do_lower_case=_UpperCamelCase,				unk_token=_UpperCamelCase,				sep_token=_UpperCamelCase,				pad_token=_UpperCamelCase,				cls_token=_UpperCamelCase,				mask_token=_UpperCamelCase,				tokenize_chinese_chars=_UpperCamelCase,				strip_accents=_UpperCamelCase,				**_UpperCamelCase,				)

        lowercase__				 =							json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
        if (
            normalizer_state.get("lowercase",				_UpperCamelCase  ) != do_lower_case
            or normalizer_state.get("strip_accents",				_UpperCamelCase  ) != strip_accents
            or normalizer_state.get("handle_chinese_chars",				_UpperCamelCase  ) != tokenize_chinese_chars
        ):
             lowercase__				 =							getattr(_UpperCamelCase,				normalizer_state.pop("type"  )  )
             lowercase__				 =							do_lower_case
             lowercase__				 =							strip_accents
             lowercase__				 =							tokenize_chinese_chars
             lowercase__				 =							normalizer_class(**_UpperCamelCase  )

        lowercase__				 =							do_lower_case


   def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase=None  ):
        '''simple docstring'''



        lowercase__				 =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

        return output


   def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase = None  ):
        '''simple docstring'''



        lowercase__				 =							[self.sep_token_id]
        lowercase__				 =							[self.cls_token_id]
        if token_ids_a is None:
             return len(cls + token_ids_a + sep  ) * [0]
        return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]


   def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase = None  ):
        '''simple docstring'''



        lowercase__				 =							self._tokenizer.model.save(_UpperCamelCase,				name=_UpperCamelCase  )
        return tuple(_UpperCamelCase  )





"""simple docstring"""


import unittest

from transformers import AutoConfig, AutoTokenizer, BertConfig, TensorType, is_flax_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, slow


if is_flax_available():
 import jax

 from transformers.models.auto.modeling_flax_auto import FlaxAutoModel
 from transformers.models.bert.modeling_flax_bert import FlaxBertModel
 from transformers.models.roberta.modeling_flax_roberta import FlaxRobertaModel






@require_flax
class    a__    (   unittest.TestCase	):


   @slow
   def 	snake_case__	(     self  ):
        '''simple docstring'''



        for model_name in ["bert-base-cased", "bert-large-uncased"]:
             with self.subTest(_UpperCAmelCase  ):
                  lowercase__				 =							AutoConfig.from_pretrained(_UpperCAmelCase  )
                  self.assertIsNotNone(_UpperCAmelCase  )
                  self.assertIsInstance(_UpperCAmelCase,				_UpperCAmelCase  )

                  lowercase__				 =							FlaxAutoModel.from_pretrained(_UpperCAmelCase  )
                  self.assertIsNotNone(_UpperCAmelCase  )
                  self.assertIsInstance(_UpperCAmelCase,				_UpperCAmelCase  )


   @slow
   def 	snake_case__	(     self  ):
        '''simple docstring'''



        for model_name in ["roberta-base", "roberta-large"]:
             with self.subTest(_UpperCAmelCase  ):
                  lowercase__				 =							AutoConfig.from_pretrained(_UpperCAmelCase  )
                  self.assertIsNotNone(_UpperCAmelCase  )
                  self.assertIsInstance(_UpperCAmelCase,				_UpperCAmelCase  )

                  lowercase__				 =							FlaxAutoModel.from_pretrained(_UpperCAmelCase  )
                  self.assertIsNotNone(_UpperCAmelCase  )
                  self.assertIsInstance(_UpperCAmelCase,				_UpperCAmelCase  )


   @slow
   def 	snake_case__	(     self  ):
        '''simple docstring'''



        for model_name in ["bert-base-cased", "bert-large-uncased"]:
             lowercase__				 =							AutoTokenizer.from_pretrained(_UpperCAmelCase  )
             lowercase__				 =							FlaxBertModel.from_pretrained(_UpperCAmelCase  )
             lowercase__				 =							tokenizer("Do you support jax jitted function?",				return_tensors=TensorType.JAX  )

             @jax.jit
             def eval(**_UpperCAmelCase  ):
                  return model(**_UpperCAmelCase  )

             eval(**_UpperCAmelCase  ).block_until_ready()


   @slow
   def 	snake_case__	(     self  ):
        '''simple docstring'''



        for model_name in ["roberta-base", "roberta-large"]:
             lowercase__				 =							AutoTokenizer.from_pretrained(_UpperCAmelCase  )
             lowercase__				 =							FlaxRobertaModel.from_pretrained(_UpperCAmelCase  )
             lowercase__				 =							tokenizer("Do you support jax jitted function?",				return_tensors=TensorType.JAX  )

             @jax.jit
             def eval(**_UpperCAmelCase  ):
                  return model(**_UpperCAmelCase  )

             eval(**_UpperCAmelCase  ).block_until_ready()


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        with self.assertRaisesRegex(
            _UpperCAmelCase,				"bert-base is not a local folder and is not a valid model identifier"  ):
             lowercase__				 =							FlaxAutoModel.from_pretrained("bert-base"  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        with self.assertRaisesRegex(
            _UpperCAmelCase,				R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"  ):
             lowercase__				 =							FlaxAutoModel.from_pretrained(_UpperCAmelCase,				revision="aaaaaa"  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        with self.assertRaisesRegex(
            _UpperCAmelCase,				"hf-internal-testing/config-no-model does not appear to have a file named flax_model.msgpack",				):
             lowercase__				 =							FlaxAutoModel.from_pretrained("hf-internal-testing/config-no-model"  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        with self.assertRaisesRegex(_UpperCAmelCase,				"Use `from_pt=True` to load this model"  ):
             lowercase__				 =							FlaxAutoModel.from_pretrained("hf-internal-testing/tiny-bert-pt-only"  )




