



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
							from test_module.custom_tokenization_fast import CustomTokenizerFast







class     __lowerCAmelCase					(    unittest.TestCase		):



					def 					_lowerCamelCase (		self     :						List[Any])     ->					str:



								"""simple docstring"""






								_UpperCAmelCase	 =     mock.Mock()
								_UpperCAmelCase	 =     5_00
								_UpperCAmelCase	 =     {}
								_UpperCAmelCase	 =     HTTPError
								_UpperCAmelCase	 =     {}

								# Download this model to make sure it's in the cache.
								_UpperCAmelCase	 =     BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('requests.Session.request'				,	return_value=A) as mock_head:
											_UpperCAmelCase	 =     BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
											# This check we did call the fake head request
											mock_head.assert_called()



					@require_tokenizers
					def 					_lowerCamelCase (		self     :						Optional[int])     ->					int:



								"""simple docstring"""






								_UpperCAmelCase	 =     mock.Mock()
								_UpperCAmelCase	 =     5_00
								_UpperCAmelCase	 =     {}
								_UpperCAmelCase	 =     HTTPError
								_UpperCAmelCase	 =     {}

								# Download this model to make sure it's in the cache.
								_UpperCAmelCase	 =     GPTaTokenizerFast.from_pretrained('gpt2')

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('requests.Session.request'				,	return_value=A) as mock_head:
											_UpperCAmelCase	 =     GPTaTokenizerFast.from_pretrained('gpt2')
											# This check we did call the fake head request
											mock_head.assert_called()



					def 					_lowerCamelCase (		self     :						Optional[int])     ->					str:



								"""simple docstring"""






								try:
											_UpperCAmelCase	 =     tempfile.mktemp()
											with open(A				,	'wb') as f:
														http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'				,	A)

											_UpperCAmelCase	 =     AlbertTokenizer.from_pretrained(A)
								finally:
											os.remove(A)

								# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
								# the current folder and have the right name.
								if os.path.isfile('tokenizer.json'):
											# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
											return
								try:
											with open('tokenizer.json'				,	'wb') as f:
														http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'				,	A)
											_UpperCAmelCase	 =     AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2')
											# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
											self.assertEqual(tokenizer.vocab_size				,	10_00)
											# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

								finally:
											os.remove('tokenizer.json')



					def 					_lowerCamelCase (		self     :						List[str])     ->					int:



								"""simple docstring"""






								_UpperCAmelCase	 =     AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model')







@is_staging_test
class     __lowerCAmelCase					(    unittest.TestCase		):
					UpperCamelCase           =  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']



					@classmethod
					def 					_lowerCamelCase (		cls     :						List[Any])     ->					List[Any]:



								"""simple docstring"""






								_UpperCAmelCase	 =     TOKEN
								HfFolder.save_token(A)



					@classmethod
					def 					_lowerCamelCase (		cls     :						List[str])     ->					Dict:



								"""simple docstring"""






								try:
											delete_repo(token=cls._token				,	repo_id='test-tokenizer')
								except HTTPError:
											pass

								try:
											delete_repo(token=cls._token				,	repo_id='valid_org/test-tokenizer-org')
								except HTTPError:
											pass

								try:
											delete_repo(token=cls._token				,	repo_id='test-dynamic-tokenizer')
								except HTTPError:
											pass



					def 					_lowerCamelCase (		self     :						List[Any])     ->					Optional[int]:



								"""simple docstring"""






								with tempfile.TemporaryDirectory() as tmp_dir:
											_UpperCAmelCase	 =     os.path.join(A				,	'vocab.txt')
											with open(A				,	'w'				,	encoding='utf-8') as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
											_UpperCAmelCase	 =     BertTokenizer(A)

								tokenizer.push_to_hub('test-tokenizer'				,	use_auth_token=self._token)
								_UpperCAmelCase	 =     BertTokenizer.from_pretrained(F"{USER}/test-tokenizer")
								self.assertDictEqual(new_tokenizer.vocab				,	tokenizer.vocab)

								# Reset repo
								delete_repo(token=self._token				,	repo_id='test-tokenizer')

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(A				,	repo_id='test-tokenizer'				,	push_to_hub=A				,	use_auth_token=self._token)

								_UpperCAmelCase	 =     BertTokenizer.from_pretrained(F"{USER}/test-tokenizer")
								self.assertDictEqual(new_tokenizer.vocab				,	tokenizer.vocab)



					def 					_lowerCamelCase (		self     :						Any)     ->					Optional[Any]:



								"""simple docstring"""






								with tempfile.TemporaryDirectory() as tmp_dir:
											_UpperCAmelCase	 =     os.path.join(A				,	'vocab.txt')
											with open(A				,	'w'				,	encoding='utf-8') as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
											_UpperCAmelCase	 =     BertTokenizer(A)

								tokenizer.push_to_hub('valid_org/test-tokenizer-org'				,	use_auth_token=self._token)
								_UpperCAmelCase	 =     BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
								self.assertDictEqual(new_tokenizer.vocab				,	tokenizer.vocab)

								# Reset repo
								delete_repo(token=self._token				,	repo_id='valid_org/test-tokenizer-org')

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(
											    A				,	repo_id='valid_org/test-tokenizer-org'				,	push_to_hub=A				,	use_auth_token=self._token)

								_UpperCAmelCase	 =     BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
								self.assertDictEqual(new_tokenizer.vocab				,	tokenizer.vocab)



					@require_tokenizers
					def 					_lowerCamelCase (		self     :						int)     ->					int:



								"""simple docstring"""






								CustomTokenizer.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
											_UpperCAmelCase	 =     os.path.join(A				,	'vocab.txt')
											with open(A				,	'w'				,	encoding='utf-8') as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
											_UpperCAmelCase	 =     CustomTokenizer(A)

								# No fast custom tokenizer
								tokenizer.push_to_hub('test-dynamic-tokenizer'				,	use_auth_token=self._token)

								_UpperCAmelCase	 =     AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"				,	trust_remote_code=A)
								# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__				,	'CustomTokenizer')

								# Fast and slow custom tokenizer
								CustomTokenizerFast.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
											_UpperCAmelCase	 =     os.path.join(A				,	'vocab.txt')
											with open(A				,	'w'				,	encoding='utf-8') as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))

											_UpperCAmelCase	 =     BertTokenizerFast.from_pretrained(A)
											bert_tokenizer.save_pretrained(A)
											_UpperCAmelCase	 =     CustomTokenizerFast.from_pretrained(A)

								tokenizer.push_to_hub('test-dynamic-tokenizer'				,	use_auth_token=self._token)

								_UpperCAmelCase	 =     AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"				,	trust_remote_code=A)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__				,	'CustomTokenizerFast')
								_UpperCAmelCase	 =     AutoTokenizer.from_pretrained(
								    F"{USER}/test-dynamic-tokenizer"				,	use_fast=A				,	trust_remote_code=A)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__				,	'CustomTokenizer')







class     __lowerCAmelCase					(    unittest.TestCase		):



					def 					_lowerCamelCase (		self     :						List[Any])     ->					Any:



								"""simple docstring"""






								_UpperCAmelCase	 =     Trie()
								trie.add('Hello 友達')
								self.assertEqual(trie.data				,	{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}})
								trie.add('Hello')
								trie.data
								self.assertEqual(trie.data				,	{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}})



					def 					_lowerCamelCase (		self     :						Tuple)     ->					str:



								"""simple docstring"""






								_UpperCAmelCase	 =     Trie()
								self.assertEqual(trie.split('[CLS] This is a extra_id_100')				,	['[CLS] This is a extra_id_100'])
								trie.add('[CLS]')
								trie.add('extra_id_1')
								trie.add('extra_id_100')
								self.assertEqual(trie.split('[CLS] This is a extra_id_100')				,	['[CLS]', ' This is a ', 'extra_id_100'])



					def 					_lowerCamelCase (		self     :						str)     ->					Optional[int]:



								"""simple docstring"""






								_UpperCAmelCase	 =     Trie()
								trie.add('A')
								self.assertEqual(trie.split('ABC')				,	['A', 'BC'])
								self.assertEqual(trie.split('BCA')				,	['BC', 'A'])



					def 					_lowerCamelCase (		self     :						List[str])     ->					int:



								"""simple docstring"""






								_UpperCAmelCase	 =     Trie()
								trie.add('TOKEN]')
								trie.add('[SPECIAL_TOKEN]')
								self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')				,	['This is something ', '[SPECIAL_TOKEN]'])



					def 					_lowerCamelCase (		self     :						Union[str, Any])     ->					Union[str, Any]:



								"""simple docstring"""






								_UpperCAmelCase	 =     Trie()
								trie.add('A')
								trie.add('P')
								trie.add('[SPECIAL_TOKEN]')
								self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')				,	['This is something ', '[SPECIAL_TOKEN]'])



					def 					_lowerCamelCase (		self     :						Dict)     ->					List[str]:



								"""simple docstring"""






								_UpperCAmelCase	 =     Trie()
								trie.add('AB')
								trie.add('B')
								trie.add('C')
								self.assertEqual(trie.split('ABC')				,	['AB', 'C'])



					def 					_lowerCamelCase (		self     :						Dict)     ->					Tuple:



								"""simple docstring"""






								_UpperCAmelCase	 =     Trie()
								trie.add('ABC')
								trie.add('B')
								trie.add('CD')
								self.assertEqual(trie.split('ABCD')				,	['ABC', 'D'])



					def 					_lowerCamelCase (		self     :						int)     ->					Dict:



								"""simple docstring"""






								_UpperCAmelCase	 =     Trie()
								_UpperCAmelCase	 =     trie.cut_text('ABC'				,	[0, 0, 2, 1, 2, 3])
								self.assertEqual(A				,	['AB', 'C'])





import random
import unittest

import torch

from diffusers import IFInpaintingPipeline
from diffusers.utils import floats_tensor
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import skip_mps, torch_device

from ..pipeline_params import (
    TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_INPAINTING_PARAMS,
)
from ..test_pipelines_common import PipelineTesterMixin
from . import IFPipelineTesterMixin







@skip_mps
class     __lowerCAmelCase					(    A       ,		A       ,		unittest.TestCase		):
			UpperCamelCase           =  IFInpaintingPipeline
			UpperCamelCase           =  TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {'''width''', '''height'''}
			UpperCamelCase           =  TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS
			UpperCamelCase           =  PipelineTesterMixin.required_optional_params - {'''latents'''}



			def 					_lowerCamelCase (		self     :						List[str])     ->					Tuple:



						"""simple docstring"""






						return self._get_dummy_components()



			def 					_lowerCamelCase (		self     :						Any				,	A     :						int				,	A     :						Dict=0)     ->					Tuple:



						"""simple docstring"""






						if str(A).startswith('mps'):
									_UpperCAmelCase	 =     torch.manual_seed(A)
						else:
									_UpperCAmelCase	 =     torch.Generator(device=A).manual_seed(A)

						_UpperCAmelCase	 =     floats_tensor((1, 3, 32, 32)				,	rng=random.Random(A)).to(A)
						_UpperCAmelCase	 =     floats_tensor((1, 3, 32, 32)				,	rng=random.Random(A)).to(A)

						_UpperCAmelCase	 =     {
						    'prompt': 'A painting of a squirrel eating a burger',
						    'image': image,
						    'mask_image': mask_image,
						    'generator': generator,
						    'num_inference_steps': 2,
						    'output_type': 'numpy',
						}

						return inputs



			@unittest.skipIf(
			    torch_device != 'cuda' or not is_xformers_available()				,	reason='XFormers attention is only available with CUDA and `xformers` installed'				,	)
			def 					_lowerCamelCase (		self     :						List[str])     ->					Union[str, Any]:



						"""simple docstring"""






						self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1E-3)



			def 					_lowerCamelCase (		self     :						Optional[Any])     ->					Optional[int]:



						"""simple docstring"""






						self._test_save_load_optional_components()



			@unittest.skipIf(torch_device != 'cuda'				,	reason='float16 requires CUDA')
			def 					_lowerCamelCase (		self     :						List[str])     ->					Any:



						"""simple docstring"""






						super().test_save_load_floataa(expected_max_diff=1E-1)



			def 					_lowerCamelCase (		self     :						Optional[int])     ->					Tuple:



						"""simple docstring"""






						self._test_attention_slicing_forward_pass(expected_max_diff=1E-2)



			def 					_lowerCamelCase (		self     :						str)     ->					List[str]:



						"""simple docstring"""






						self._test_save_load_local()



			def 					_lowerCamelCase (		self     :						int)     ->					Tuple:



						"""simple docstring"""






						self._test_inference_batch_single_identical(
						    expected_max_diff=1E-2				,	)
