

import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast







class 	lowercase    (				unittest.TestCase							):


  """simple docstring"""




  def 		A__					(				self):
   _UpperCamelCase   :		List[str]			=						mock.Mock()
   _UpperCamelCase   :		Any			=						5_00
   _UpperCamelCase   :		int			=						{}
   _UpperCamelCase   :		Union[str, Any]			=						HTTPError
   _UpperCamelCase   :		int			=						{}

   # Download this model to make sure it's in the cache.
   _UpperCamelCase   :		Union[str, Any]			=						BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')

   # Under the mock environment we get a 500 error when trying to reach the tokenizer.
   with mock.patch('requests.Session.request'	,       return_value=lowerCamelCase_) as mock_head:
    _UpperCamelCase   :		List[str]			=						BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
    # This check we did call the fake head request
    mock_head.assert_called()


  @require_tokenizers
  def 		A__					(				self):
   _UpperCamelCase   :		List[str]			=						mock.Mock()
   _UpperCamelCase   :		Any			=						5_00
   _UpperCamelCase   :		Any			=						{}
   _UpperCamelCase   :		Union[str, Any]			=						HTTPError
   _UpperCamelCase   :		Optional[Any]			=						{}

   # Download this model to make sure it's in the cache.
   _UpperCamelCase   :		Optional[Any]			=						GPTaTokenizerFast.from_pretrained('gpt2')

   # Under the mock environment we get a 500 error when trying to reach the tokenizer.
   with mock.patch('requests.Session.request'	,       return_value=lowerCamelCase_) as mock_head:
    _UpperCamelCase   :		Optional[Any]			=						GPTaTokenizerFast.from_pretrained('gpt2')
    # This check we did call the fake head request
    mock_head.assert_called()


  def 		A__					(				self):




   try:
    _UpperCamelCase   :		Optional[int]			=						tempfile.mktemp()
    with open(lowerCamelCase_	,       'wb') as f:
     http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'	,       lowerCamelCase_)

    _UpperCamelCase   :		List[str]			=						AlbertTokenizer.from_pretrained(lowerCamelCase_)
   finally:
    os.remove(lowerCamelCase_)

   # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
   # the current folder and have the right name.
   if os.path.isfile('tokenizer.json'):
    # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
    return
   try:
    with open('tokenizer.json'	,       'wb') as f:
     http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'	,       lowerCamelCase_)
    _UpperCamelCase   :		str			=						AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2')
    # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
    self.assertEqual(tokenizer.vocab_size	,       10_00)
    # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

   finally:
    os.remove('tokenizer.json')






  def 		A__					(				self):
   _UpperCamelCase   :		Optional[Any]			=						AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model')







@is_staging_test
class 	lowercase    (				unittest.TestCase							):


  """simple docstring"""




  a__					       =  ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]


  @classmethod
  def 		A__					(				cls):
   _UpperCamelCase   :		str			=						TOKEN
   HfFolder.save_token(lowerCamelCase_)


  @classmethod
  def 		A__					(				cls):




   try:
    delete_repo(token=cls._token	,       repo_id='test-tokenizer')
   except HTTPError:
    pass

   try:
    delete_repo(token=cls._token	,       repo_id='valid_org/test-tokenizer-org')
   except HTTPError:
    pass

   try:
    delete_repo(token=cls._token	,       repo_id='test-dynamic-tokenizer')
   except HTTPError:
    pass


  def 		A__					(				self):




   with tempfile.TemporaryDirectory() as tmp_dir:
    _UpperCamelCase   :		Union[str, Any]			=						os.path.join(lowerCamelCase_	,       'vocab.txt')
    with open(lowerCamelCase_	,       'w'	,       encoding='utf-8') as vocab_writer:
     vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
    _UpperCamelCase   :		Tuple			=						BertTokenizer(lowerCamelCase_)

   tokenizer.push_to_hub('test-tokenizer'	,       use_auth_token=self._token)
   _UpperCamelCase   :		Optional[Any]			=						BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer''')
   self.assertDictEqual(new_tokenizer.vocab	,       tokenizer.vocab)

   # Reset repo
   delete_repo(token=self._token	,       repo_id='test-tokenizer')

   # Push to hub via save_pretrained
   with tempfile.TemporaryDirectory() as tmp_dir:
    tokenizer.save_pretrained(lowerCamelCase_	,       repo_id='test-tokenizer'	,       push_to_hub=lowerCamelCase_	,       use_auth_token=self._token)

   _UpperCamelCase   :		Optional[int]			=						BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer''')
   self.assertDictEqual(new_tokenizer.vocab	,       tokenizer.vocab)


  def 		A__					(				self):




   with tempfile.TemporaryDirectory() as tmp_dir:
    _UpperCamelCase   :		int			=						os.path.join(lowerCamelCase_	,       'vocab.txt')
    with open(lowerCamelCase_	,       'w'	,       encoding='utf-8') as vocab_writer:
     vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
    _UpperCamelCase   :		Union[str, Any]			=						BertTokenizer(lowerCamelCase_)

   tokenizer.push_to_hub('valid_org/test-tokenizer-org'	,       use_auth_token=self._token)
   _UpperCamelCase   :		Dict			=						BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
   self.assertDictEqual(new_tokenizer.vocab	,       tokenizer.vocab)

   # Reset repo
   delete_repo(token=self._token	,       repo_id='valid_org/test-tokenizer-org')

   # Push to hub via save_pretrained
   with tempfile.TemporaryDirectory() as tmp_dir:
    tokenizer.save_pretrained(
        lowerCamelCase_	,       repo_id='valid_org/test-tokenizer-org'	,       push_to_hub=lowerCamelCase_	,       use_auth_token=self._token)

   _UpperCamelCase   :		Union[str, Any]			=						BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
   self.assertDictEqual(new_tokenizer.vocab	,       tokenizer.vocab)






  @require_tokenizers
  def 		A__					(				self):
   CustomTokenizer.register_for_auto_class()
   with tempfile.TemporaryDirectory() as tmp_dir:
    _UpperCamelCase   :		List[Any]			=						os.path.join(lowerCamelCase_	,       'vocab.txt')
    with open(lowerCamelCase_	,       'w'	,       encoding='utf-8') as vocab_writer:
     vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
    _UpperCamelCase   :		Optional[Any]			=						CustomTokenizer(lowerCamelCase_)

   # No fast custom tokenizer
   tokenizer.push_to_hub('test-dynamic-tokenizer'	,       use_auth_token=self._token)

   _UpperCamelCase   :		Optional[Any]			=						AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''	,       trust_remote_code=lowerCamelCase_)
   # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
   self.assertEqual(tokenizer.__class__.__name__	,       'CustomTokenizer')

   # Fast and slow custom tokenizer
   CustomTokenizerFast.register_for_auto_class()
   with tempfile.TemporaryDirectory() as tmp_dir:
    _UpperCamelCase   :		Dict			=						os.path.join(lowerCamelCase_	,       'vocab.txt')
    with open(lowerCamelCase_	,       'w'	,       encoding='utf-8') as vocab_writer:
     vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))

    _UpperCamelCase   :		Dict			=						BertTokenizerFast.from_pretrained(lowerCamelCase_)
    bert_tokenizer.save_pretrained(lowerCamelCase_)
    _UpperCamelCase   :		List[Any]			=						CustomTokenizerFast.from_pretrained(lowerCamelCase_)

   tokenizer.push_to_hub('test-dynamic-tokenizer'	,       use_auth_token=self._token)

   _UpperCamelCase   :		List[Any]			=						AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''	,       trust_remote_code=lowerCamelCase_)
   # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
   self.assertEqual(tokenizer.__class__.__name__	,       'CustomTokenizerFast')
   _UpperCamelCase   :		List[Any]			=						AutoTokenizer.from_pretrained(
       f'''{USER}/test-dynamic-tokenizer'''	,       use_fast=lowerCamelCase_	,       trust_remote_code=lowerCamelCase_)
   # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
   self.assertEqual(tokenizer.__class__.__name__	,       'CustomTokenizer')







class 	lowercase    (				unittest.TestCase							):


  """simple docstring"""




  def 		A__					(				self):
   _UpperCamelCase   :		Union[str, Any]			=						Trie()
   trie.add('Hello 友達')
   self.assertEqual(trie.data	,       {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}})
   trie.add('Hello')
   trie.data
   self.assertEqual(trie.data	,       {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}})


  def 		A__					(				self):
   _UpperCamelCase   :		Union[str, Any]			=						Trie()
   self.assertEqual(trie.split('[CLS] This is a extra_id_100')	,       ['[CLS] This is a extra_id_100'])
   trie.add('[CLS]')
   trie.add('extra_id_1')
   trie.add('extra_id_100')
   self.assertEqual(trie.split('[CLS] This is a extra_id_100')	,       ['[CLS]', ' This is a ', 'extra_id_100'])


  def 		A__					(				self):
   _UpperCamelCase   :		List[Any]			=						Trie()
   trie.add('A')
   self.assertEqual(trie.split('ABC')	,       ['A', 'BC'])
   self.assertEqual(trie.split('BCA')	,       ['BC', 'A'])


  def 		A__					(				self):
   _UpperCamelCase   :		Optional[int]			=						Trie()
   trie.add('TOKEN]')
   trie.add('[SPECIAL_TOKEN]')
   self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')	,       ['This is something ', '[SPECIAL_TOKEN]'])


  def 		A__					(				self):
   _UpperCamelCase   :		Optional[Any]			=						Trie()
   trie.add('A')
   trie.add('P')
   trie.add('[SPECIAL_TOKEN]')
   self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')	,       ['This is something ', '[SPECIAL_TOKEN]'])


  def 		A__					(				self):
   _UpperCamelCase   :		List[Any]			=						Trie()
   trie.add('AB')
   trie.add('B')
   trie.add('C')
   self.assertEqual(trie.split('ABC')	,       ['AB', 'C'])


  def 		A__					(				self):
   _UpperCamelCase   :		List[Any]			=						Trie()
   trie.add('ABC')
   trie.add('B')
   trie.add('CD')
   self.assertEqual(trie.split('ABCD')	,       ['ABC', 'D'])






  def 		A__					(				self):
   _UpperCamelCase   :		int			=						Trie()
   _UpperCamelCase   :		Optional[int]			=						trie.cut_text('ABC'	,       [0, 0, 2, 1, 2, 3])
   self.assertEqual(lowerCamelCase_	,       ['AB', 'C'])







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCAmelCase__								=		logging.get_logger(__name__)

lowerCAmelCase__								=		{
    """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/config.json""",
    """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/config.json""",
    """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/config.json""",
    """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/config.json""",
    """bert-base-multilingual-uncased""": """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/config.json""",
    """bert-base-multilingual-cased""": """https://huggingface.co/bert-base-multilingual-cased/resolve/main/config.json""",
    """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/config.json""",
    """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/config.json""",
    """bert-large-uncased-whole-word-masking""": (
        """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/config.json"""
    ),
    """bert-large-cased-whole-word-masking""": (
        """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/config.json"""
    ),
    """bert-large-uncased-whole-word-masking-finetuned-squad""": (
        """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/config.json"""
    ),
    """bert-large-cased-whole-word-masking-finetuned-squad""": (
        """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/config.json"""
    ),
    """bert-base-cased-finetuned-mrpc""": """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/config.json""",
    """bert-base-german-dbmdz-cased""": """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/config.json""",
    """bert-base-german-dbmdz-uncased""": """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/config.json""",
    """cl-tohoku/bert-base-japanese""": """https://huggingface.co/cl-tohoku/bert-base-japanese/resolve/main/config.json""",
    """cl-tohoku/bert-base-japanese-whole-word-masking""": (
        """https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking/resolve/main/config.json"""
    ),
    """cl-tohoku/bert-base-japanese-char""": (
        """https://huggingface.co/cl-tohoku/bert-base-japanese-char/resolve/main/config.json"""
    ),
    """cl-tohoku/bert-base-japanese-char-whole-word-masking""": (
        """https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking/resolve/main/config.json"""
    ),
    """TurkuNLP/bert-base-finnish-cased-v1""": (
        """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/config.json"""
    ),
    """TurkuNLP/bert-base-finnish-uncased-v1""": (
        """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/config.json"""
    ),
    """wietsedv/bert-base-dutch-cased""": """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/config.json""",
    # See all BERT models at https://huggingface.co/models?filter=bert
}







class 	lowercase    (				_lowercase							):


		"""simple docstring"""




		a__					       =  "bert"


		def __init__(				self	,       __snake_case=3_05_22	,       __snake_case=7_68	,       __snake_case=12	,       __snake_case=12	,       __snake_case=30_72	,       __snake_case="gelu"	,       __snake_case=0.1	,       __snake_case=0.1	,       __snake_case=5_12	,       __snake_case=2	,       __snake_case=0.0_2	,       __snake_case=1e-12	,       __snake_case=0	,       __snake_case="absolute"	,       __snake_case=True	,       __snake_case=None	,       **__snake_case	,       ):
			super().__init__(pad_token_id=__snake_case	,       **__snake_case)

			_UpperCamelCase   :		int			=						vocab_size
			_UpperCamelCase   :		Optional[Any]			=						hidden_size
			_UpperCamelCase   :		Optional[Any]			=						num_hidden_layers
			_UpperCamelCase   :		List[str]			=						num_attention_heads
			_UpperCamelCase   :		int			=						hidden_act
			_UpperCamelCase   :		Optional[Any]			=						intermediate_size
			_UpperCamelCase   :		Union[str, Any]			=						hidden_dropout_prob
			_UpperCamelCase   :		Tuple			=						attention_probs_dropout_prob
			_UpperCamelCase   :		Optional[int]			=						max_position_embeddings
			_UpperCamelCase   :		str			=						type_vocab_size
			_UpperCamelCase   :		Optional[Any]			=						initializer_range
			_UpperCamelCase   :		List[str]			=						layer_norm_eps
			_UpperCamelCase   :		Any			=						position_embedding_type
			_UpperCamelCase   :		Any			=						use_cache
			_UpperCamelCase   :		Any			=						classifier_dropout







class 	lowercase    (				_lowercase							):


		"""simple docstring"""




		@property
		def 		A__					(				self):
			if self.task == "multiple-choice":
				_UpperCamelCase   :		Union[str, Any]			=						{0: 'batch', 1: 'choice', 2: 'sequence'}
			else:
				_UpperCamelCase   :		Optional[Any]			=						{0: 'batch', 1: 'sequence'}
			return OrderedDict(
			    [
			        ('input_ids', dynamic_axis),
			        ('attention_mask', dynamic_axis),
			        ('token_type_ids', dynamic_axis),
			    ])




