



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
							from test_module.custom_tokenization_fast import CustomTokenizerFast







class     __lowerCAmelCase					(    unittest.TestCase		):



			def 					_lowerCamelCase (		self     :						List[Any])     ->					str:



						"""simple docstring"""






						_UpperCAmelCase	 =     mock.Mock()
						_UpperCAmelCase	 =     5_00
						_UpperCAmelCase	 =     {}
						_UpperCAmelCase	 =     HTTPError
						_UpperCAmelCase	 =     {}

						# Download this model to make sure it's in the cache.
						_UpperCAmelCase	 =     BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')

						# Under the mock environment we get a 500 error when trying to reach the tokenizer.
						with mock.patch('requests.Session.request'				,	return_value=A) as mock_head:
									_UpperCAmelCase	 =     BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
									# This check we did call the fake head request
									mock_head.assert_called()



			@require_tokenizers
			def 					_lowerCamelCase (		self     :						Optional[int])     ->					int:



						"""simple docstring"""






						_UpperCAmelCase	 =     mock.Mock()
						_UpperCAmelCase	 =     5_00
						_UpperCAmelCase	 =     {}
						_UpperCAmelCase	 =     HTTPError
						_UpperCAmelCase	 =     {}

						# Download this model to make sure it's in the cache.
						_UpperCAmelCase	 =     GPTaTokenizerFast.from_pretrained('gpt2')

						# Under the mock environment we get a 500 error when trying to reach the tokenizer.
						with mock.patch('requests.Session.request'				,	return_value=A) as mock_head:
									_UpperCAmelCase	 =     GPTaTokenizerFast.from_pretrained('gpt2')
									# This check we did call the fake head request
									mock_head.assert_called()



			def 					_lowerCamelCase (		self     :						Optional[int])     ->					str:



						"""simple docstring"""






						try:
									_UpperCAmelCase	 =     tempfile.mktemp()
									with open(A				,	'wb') as f:
												http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'				,	A)

									_UpperCAmelCase	 =     AlbertTokenizer.from_pretrained(A)
						finally:
									os.remove(A)

						# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
						# the current folder and have the right name.
						if os.path.isfile('tokenizer.json'):
									# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
									return
						try:
									with open('tokenizer.json'				,	'wb') as f:
												http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'				,	A)
									_UpperCAmelCase	 =     AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2')
									# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
									self.assertEqual(tokenizer.vocab_size				,	10_00)
									# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

						finally:
									os.remove('tokenizer.json')



			def 					_lowerCamelCase (		self     :						List[str])     ->					int:



						"""simple docstring"""






						_UpperCAmelCase	 =     AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model')







@is_staging_test
class     __lowerCAmelCase					(    unittest.TestCase		):
			UpperCamelCase           =  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']



			@classmethod
			def 					_lowerCamelCase (		cls     :						List[Any])     ->					List[Any]:



						"""simple docstring"""






						_UpperCAmelCase	 =     TOKEN
						HfFolder.save_token(A)



			@classmethod
			def 					_lowerCamelCase (		cls     :						List[str])     ->					Dict:



						"""simple docstring"""






						try:
									delete_repo(token=cls._token				,	repo_id='test-tokenizer')
						except HTTPError:
									pass

						try:
									delete_repo(token=cls._token				,	repo_id='valid_org/test-tokenizer-org')
						except HTTPError:
									pass

						try:
									delete_repo(token=cls._token				,	repo_id='test-dynamic-tokenizer')
						except HTTPError:
									pass



			def 					_lowerCamelCase (		self     :						List[Any])     ->					Optional[int]:



						"""simple docstring"""






						with tempfile.TemporaryDirectory() as tmp_dir:
									_UpperCAmelCase	 =     os.path.join(A				,	'vocab.txt')
									with open(A				,	'w'				,	encoding='utf-8') as vocab_writer:
												vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
									_UpperCAmelCase	 =     BertTokenizer(A)

						tokenizer.push_to_hub('test-tokenizer'				,	use_auth_token=self._token)
						_UpperCAmelCase	 =     BertTokenizer.from_pretrained(F"{USER}/test-tokenizer")
						self.assertDictEqual(new_tokenizer.vocab				,	tokenizer.vocab)

						# Reset repo
						delete_repo(token=self._token				,	repo_id='test-tokenizer')

						# Push to hub via save_pretrained
						with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(A				,	repo_id='test-tokenizer'				,	push_to_hub=A				,	use_auth_token=self._token)

						_UpperCAmelCase	 =     BertTokenizer.from_pretrained(F"{USER}/test-tokenizer")
						self.assertDictEqual(new_tokenizer.vocab				,	tokenizer.vocab)



			def 					_lowerCamelCase (		self     :						Any)     ->					Optional[Any]:



						"""simple docstring"""






						with tempfile.TemporaryDirectory() as tmp_dir:
									_UpperCAmelCase	 =     os.path.join(A				,	'vocab.txt')
									with open(A				,	'w'				,	encoding='utf-8') as vocab_writer:
												vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
									_UpperCAmelCase	 =     BertTokenizer(A)

						tokenizer.push_to_hub('valid_org/test-tokenizer-org'				,	use_auth_token=self._token)
						_UpperCAmelCase	 =     BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
						self.assertDictEqual(new_tokenizer.vocab				,	tokenizer.vocab)

						# Reset repo
						delete_repo(token=self._token				,	repo_id='valid_org/test-tokenizer-org')

						# Push to hub via save_pretrained
						with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(
									    A				,	repo_id='valid_org/test-tokenizer-org'				,	push_to_hub=A				,	use_auth_token=self._token)

						_UpperCAmelCase	 =     BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
						self.assertDictEqual(new_tokenizer.vocab				,	tokenizer.vocab)



			@require_tokenizers
			def 					_lowerCamelCase (		self     :						int)     ->					int:



						"""simple docstring"""






						CustomTokenizer.register_for_auto_class()
						with tempfile.TemporaryDirectory() as tmp_dir:
									_UpperCAmelCase	 =     os.path.join(A				,	'vocab.txt')
									with open(A				,	'w'				,	encoding='utf-8') as vocab_writer:
												vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
									_UpperCAmelCase	 =     CustomTokenizer(A)

						# No fast custom tokenizer
						tokenizer.push_to_hub('test-dynamic-tokenizer'				,	use_auth_token=self._token)

						_UpperCAmelCase	 =     AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"				,	trust_remote_code=A)
						# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
						self.assertEqual(tokenizer.__class__.__name__				,	'CustomTokenizer')

						# Fast and slow custom tokenizer
						CustomTokenizerFast.register_for_auto_class()
						with tempfile.TemporaryDirectory() as tmp_dir:
									_UpperCAmelCase	 =     os.path.join(A				,	'vocab.txt')
									with open(A				,	'w'				,	encoding='utf-8') as vocab_writer:
												vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))

									_UpperCAmelCase	 =     BertTokenizerFast.from_pretrained(A)
									bert_tokenizer.save_pretrained(A)
									_UpperCAmelCase	 =     CustomTokenizerFast.from_pretrained(A)

						tokenizer.push_to_hub('test-dynamic-tokenizer'				,	use_auth_token=self._token)

						_UpperCAmelCase	 =     AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"				,	trust_remote_code=A)
						# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
						self.assertEqual(tokenizer.__class__.__name__				,	'CustomTokenizerFast')
						_UpperCAmelCase	 =     AutoTokenizer.from_pretrained(
						    F"{USER}/test-dynamic-tokenizer"				,	use_fast=A				,	trust_remote_code=A)
						# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
						self.assertEqual(tokenizer.__class__.__name__				,	'CustomTokenizer')







class     __lowerCAmelCase					(    unittest.TestCase		):



			def 					_lowerCamelCase (		self     :						List[Any])     ->					Any:



						"""simple docstring"""






						_UpperCAmelCase	 =     Trie()
						trie.add('Hello 友達')
						self.assertEqual(trie.data				,	{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}})
						trie.add('Hello')
						trie.data
						self.assertEqual(trie.data				,	{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}})



			def 					_lowerCamelCase (		self     :						Tuple)     ->					str:



						"""simple docstring"""






						_UpperCAmelCase	 =     Trie()
						self.assertEqual(trie.split('[CLS] This is a extra_id_100')				,	['[CLS] This is a extra_id_100'])
						trie.add('[CLS]')
						trie.add('extra_id_1')
						trie.add('extra_id_100')
						self.assertEqual(trie.split('[CLS] This is a extra_id_100')				,	['[CLS]', ' This is a ', 'extra_id_100'])



			def 					_lowerCamelCase (		self     :						str)     ->					Optional[int]:



						"""simple docstring"""






						_UpperCAmelCase	 =     Trie()
						trie.add('A')
						self.assertEqual(trie.split('ABC')				,	['A', 'BC'])
						self.assertEqual(trie.split('BCA')				,	['BC', 'A'])



			def 					_lowerCamelCase (		self     :						List[str])     ->					int:



						"""simple docstring"""






						_UpperCAmelCase	 =     Trie()
						trie.add('TOKEN]')
						trie.add('[SPECIAL_TOKEN]')
						self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')				,	['This is something ', '[SPECIAL_TOKEN]'])



			def 					_lowerCamelCase (		self     :						Union[str, Any])     ->					Union[str, Any]:



						"""simple docstring"""






						_UpperCAmelCase	 =     Trie()
						trie.add('A')
						trie.add('P')
						trie.add('[SPECIAL_TOKEN]')
						self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')				,	['This is something ', '[SPECIAL_TOKEN]'])



			def 					_lowerCamelCase (		self     :						Dict)     ->					List[str]:



						"""simple docstring"""






						_UpperCAmelCase	 =     Trie()
						trie.add('AB')
						trie.add('B')
						trie.add('C')
						self.assertEqual(trie.split('ABC')				,	['AB', 'C'])



			def 					_lowerCamelCase (		self     :						Dict)     ->					Tuple:



						"""simple docstring"""






						_UpperCAmelCase	 =     Trie()
						trie.add('ABC')
						trie.add('B')
						trie.add('CD')
						self.assertEqual(trie.split('ABCD')				,	['ABC', 'D'])



			def 					_lowerCamelCase (		self     :						int)     ->					Dict:



						"""simple docstring"""






						_UpperCAmelCase	 =     Trie()
						_UpperCAmelCase	 =     trie.cut_text('ABC'				,	[0, 0, 2, 1, 2, 3])
						self.assertEqual(A				,	['AB', 'C'])





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCAmelCase__     = {"configuration_yolos": ["YOLOS_PRETRAINED_CONFIG_ARCHIVE_MAP", "YolosConfig", "YolosOnnxConfig"]}

try:
							if not is_vision_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__     = ["YolosFeatureExtractor"]
							UpperCAmelCase__     = ["YolosImageProcessor"]

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__     = [
							    "YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST",
							    "YolosForObjectDetection",
							    "YolosModel",
							    "YolosPreTrainedModel",
							]


if TYPE_CHECKING:
							from .configuration_yolos import YOLOS_PRETRAINED_CONFIG_ARCHIVE_MAP, YolosConfig, YolosOnnxConfig

							try:
														if not is_vision_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .feature_extraction_yolos import YolosFeatureExtractor
														from .image_processing_yolos import YolosImageProcessor

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_yolos import (
														    YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST,
														    YolosForObjectDetection,
														    YolosModel,
														    YolosPreTrainedModel,
														)


else:
							import sys

							UpperCAmelCase__     = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
