





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


SCREAMING_SNAKE_CASE__       :			Union[str, Any]	      =						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__       :			int	      =						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__       :			str	      =						{
    """vocab_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-german-cased""": """https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt""",
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-german-cased""": (
            """https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE__       :			int	      =						{
    """distilbert-base-uncased""": 5_1_2,
    """distilbert-base-uncased-distilled-squad""": 5_1_2,
    """distilbert-base-cased""": 5_1_2,
    """distilbert-base-cased-distilled-squad""": 5_1_2,
    """distilbert-base-german-cased""": 5_1_2,
    """distilbert-base-multilingual-cased""": 5_1_2,
}


SCREAMING_SNAKE_CASE__       :			Optional[Any]	      =						{
    """distilbert-base-uncased""": {"""do_lower_case""": True},
    """distilbert-base-uncased-distilled-squad""": {"""do_lower_case""": True},
    """distilbert-base-cased""": {"""do_lower_case""": False},
    """distilbert-base-cased-distilled-squad""": {"""do_lower_case""": False},
    """distilbert-base-german-cased""": {"""do_lower_case""": False},
    """distilbert-base-multilingual-cased""": {"""do_lower_case""": False},
}







class   __lowerCAmelCase			(					_UpperCamelCase  ):
   _UpperCamelCase :						List[Any]     		=				VOCAB_FILES_NAMES
   _UpperCamelCase :						Tuple     		=				PRETRAINED_VOCAB_FILES_MAP
   _UpperCamelCase :						Tuple     		=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCamelCase :						Optional[int]     		=				PRETRAINED_INIT_CONFIGURATION
   _UpperCamelCase :						Dict     		=				["""input_ids""", """attention_mask"""]
   _UpperCamelCase :						List[Any]     		=				DistilBertTokenizer





   def __init__( self					,			snake_case=None					,			snake_case=None					,			snake_case=True					,			snake_case="[UNK]"					,			snake_case="[SEP]"					,			snake_case="[PAD]"					,			snake_case="[CLS]"					,			snake_case="[MASK]"					,			snake_case=True					,			snake_case=None					,			**snake_case					,			)				->  List[Any]:
      """simple docstring"""





      super().__init__(
          snake_case					,			tokenizer_file=snake_case					,			do_lower_case=snake_case					,			unk_token=snake_case					,			sep_token=snake_case					,			pad_token=snake_case					,			cls_token=snake_case					,			mask_token=snake_case					,			tokenize_chinese_chars=snake_case					,			strip_accents=snake_case					,			**snake_case					,			)

      a__	:						int      		=  json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
      if (
          normalizer_state.get("lowercase"					,			snake_case      ) != do_lower_case
          or normalizer_state.get("strip_accents"					,			snake_case      ) != strip_accents
          or normalizer_state.get("handle_chinese_chars"					,			snake_case      ) != tokenize_chinese_chars
      ):
         a__	:						List[Any]      		=  getattr(snake_case					,			normalizer_state.pop("type"      )      )
         a__	:						Dict      		=  do_lower_case
         a__	:						Optional[Any]      		=  strip_accents
         a__	:						Dict      		=  tokenize_chinese_chars
         a__	:						Any      		=  normalizer_class(**snake_case      )

      a__	:						List[Any]      		=  do_lower_case





   def 	_snake_case ( self					,			snake_case					,			snake_case=None      )				->  Any:
      """simple docstring"""





      a__	:						Union[str, Any]      		=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

      return output





   def 	_snake_case ( self					,			snake_case					,			snake_case = None      )				->  List[int]:
      """simple docstring"""





      a__	:						Tuple      		=  [self.sep_token_id]
      a__	:						str      		=  [self.cls_token_id]
      if token_ids_a is None:
         return len(cls + token_ids_a + sep      ) * [0]
      return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]





   def 	_snake_case ( self					,			snake_case					,			snake_case = None      )				->  Tuple[str]:
      """simple docstring"""





      a__	:						Optional[int]      		=  self._tokenizer.model.save(snake_case					,			name=snake_case      )
      return tuple(snake_case      )








import subprocess
import sys

from transformers import BertConfig, BertModel, BertTokenizer, pipeline
from transformers.testing_utils import TestCasePlus, require_torch







class   __lowerCAmelCase			(					_UpperCamelCase  ):





   @require_torch
   def 	_snake_case ( self      )				->  str:
      """simple docstring"""





      a__	:						Tuple      		=  "\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        "

      a__	:						str      		=  "\nmname = \"hf-internal-testing/tiny-random-bert\"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task=\"fill-mask\", model=mname)\nprint(\"success\")\n        "

      a__	:						Optional[Any]      		=  "\nimport socket\ndef offline_socket(*args, **kwargs): raise RuntimeError(\"Offline mode is enabled, we shouldn't access internet\")\nsocket.socket = offline_socket\n        "

      # Force fetching the files so that we can use the cache
      a__	:						Tuple      		=  "hf-internal-testing/tiny-random-bert"
      BertConfig.from_pretrained(snake_case      )
      BertModel.from_pretrained(snake_case      )
      BertTokenizer.from_pretrained(snake_case      )
      pipeline(task="fill-mask"					,			model=snake_case      )

      # baseline - just load from_pretrained with normal network
      a__	:						Optional[int]      		=  [sys.executable, "-c", "\n".join([load, run, mock]      )]

      # should succeed
      a__	:						Dict      		=  self.get_env()
      # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
      a__	:						Tuple      		=  "1"
      a__	:						List[Any]      		=  subprocess.run(snake_case					,			env=snake_case					,			check=snake_case					,			capture_output=snake_case      )
      self.assertEqual(result.returncode					,			0					,			result.stderr      )
      self.assertIn("success"					,			result.stdout.decode()      )





   @require_torch
   def 	_snake_case ( self      )				->  List[Any]:
      """simple docstring"""





      a__	:						Tuple      		=  "\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        "

      a__	:						Optional[Any]      		=  "\nmname = \"hf-internal-testing/tiny-random-bert\"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task=\"fill-mask\", model=mname)\nprint(\"success\")\n        "

      a__	:						Tuple      		=  "\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error(\"Faking flaky internet\")\nsocket.socket = offline_socket\n        "

      # Force fetching the files so that we can use the cache
      a__	:						List[Any]      		=  "hf-internal-testing/tiny-random-bert"
      BertConfig.from_pretrained(snake_case      )
      BertModel.from_pretrained(snake_case      )
      BertTokenizer.from_pretrained(snake_case      )
      pipeline(task="fill-mask"					,			model=snake_case      )

      # baseline - just load from_pretrained with normal network
      a__	:						Tuple      		=  [sys.executable, "-c", "\n".join([load, run, mock]      )]

      # should succeed
      a__	:						Any      		=  self.get_env()
      a__	:						int      		=  subprocess.run(snake_case					,			env=snake_case					,			check=snake_case					,			capture_output=snake_case      )
      self.assertEqual(result.returncode					,			0					,			result.stderr      )
      self.assertIn("success"					,			result.stdout.decode()      )





   @require_torch
   def 	_snake_case ( self      )				->  Tuple:
      """simple docstring"""





      a__	:						List[Any]      		=  "\nfrom transformers import BertConfig, BertModel, BertTokenizer\n        "

      a__	:						Dict      		=  "\nmname = \"hf-internal-testing/tiny-random-bert-sharded\"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nprint(\"success\")\n        "

      a__	:						int      		=  "\nimport socket\ndef offline_socket(*args, **kwargs): raise ValueError(\"Offline mode is enabled\")\nsocket.socket = offline_socket\n        "

      # baseline - just load from_pretrained with normal network
      a__	:						List[Any]      		=  [sys.executable, "-c", "\n".join([load, run]      )]

      # should succeed
      a__	:						str      		=  self.get_env()
      a__	:						List[str]      		=  subprocess.run(snake_case					,			env=snake_case					,			check=snake_case					,			capture_output=snake_case      )
      self.assertEqual(result.returncode					,			0					,			result.stderr      )
      self.assertIn("success"					,			result.stdout.decode()      )

      # next emulate no network
      a__	:						List[Any]      		=  [sys.executable, "-c", "\n".join([load, mock, run]      )]

      # Doesn't fail anymore since the model is in the cache due to other tests, so commenting this.
      # env["TRANSFORMERS_OFFLINE"] = "0"
      # result = subprocess.run(cmd, env=env, check=False, capture_output=True)
      # self.assertEqual(result.returncode, 1, result.stderr)

      # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
      a__	:						Union[str, Any]      		=  "1"
      a__	:						Tuple      		=  subprocess.run(snake_case					,			env=snake_case					,			check=snake_case					,			capture_output=snake_case      )
      self.assertEqual(result.returncode					,			0					,			result.stderr      )
      self.assertIn("success"					,			result.stdout.decode()      )





   @require_torch
   def 	_snake_case ( self      )				->  Tuple:
      """simple docstring"""





      a__	:						Optional[Any]      		=  "\nfrom transformers import pipeline\n        "
      a__	:						int      		=  "\nmname = \"hf-internal-testing/tiny-random-bert\"\npipe = pipeline(model=mname)\n        "

      a__	:						Dict      		=  "\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error(\"Offline mode is enabled\")\nsocket.socket = offline_socket\n        "
      a__	:						List[str]      		=  self.get_env()
      a__	:						Union[str, Any]      		=  "1"
      a__	:						List[str]      		=  [sys.executable, "-c", "\n".join([load, mock, run]      )]
      a__	:						Any      		=  subprocess.run(snake_case					,			env=snake_case					,			check=snake_case					,			capture_output=snake_case      )
      self.assertEqual(result.returncode					,			1					,			result.stderr      )
      self.assertIn(
          "You cannot infer task automatically within `pipeline` when using offline mode"					,			result.stderr.decode().replace("\n"					,			""      )					,			)





   @require_torch
   def 	_snake_case ( self      )				->  Union[str, Any]:
      """simple docstring"""





      a__	:						Any      		=  "\nfrom transformers import AutoModel\n        "
      a__	:						Any      		=  "\nmname = \"hf-internal-testing/test_dynamic_model\"\nAutoModel.from_pretrained(mname, trust_remote_code=True)\nprint(\"success\")\n        "

      # baseline - just load from_pretrained with normal network
      a__	:						List[str]      		=  [sys.executable, "-c", "\n".join([load, run]      )]

      # should succeed
      a__	:						Optional[Any]      		=  self.get_env()
      a__	:						Union[str, Any]      		=  subprocess.run(snake_case					,			env=snake_case					,			check=snake_case					,			capture_output=snake_case      )
      self.assertEqual(result.returncode					,			0					,			result.stderr      )
      self.assertIn("success"					,			result.stdout.decode()      )

      # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
      a__	:						Dict      		=  "1"
      a__	:						Union[str, Any]      		=  subprocess.run(snake_case					,			env=snake_case					,			check=snake_case					,			capture_output=snake_case      )
      self.assertEqual(result.returncode					,			0					,			result.stderr      )
      self.assertIn("success"					,			result.stdout.decode()      )

