



"""simple docstring"""
import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
			from test_module.custom_tokenization_fast import CustomTokenizerFast


class 							lowerCamelCase_(   unittest.TestCase    ):





					'''simple docstring'''



					def 			snake_case__      (    self   ):
									_lowerCamelCase        =				mock.Mock()
									_lowerCamelCase        =				5_0_0
									_lowerCamelCase        =				{}
									_lowerCamelCase        =				HTTPError
									_lowerCamelCase        =				{}

									# Download this model to make sure it's in the cache.
									_lowerCamelCase        =				BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''   )

									# Under the mock environment we get a 500 error when trying to reach the tokenizer.
									with mock.patch('''requests.Session.request'''				,       return_value=lowerCamelCase__   ) as mock_head:
													_lowerCamelCase        =				BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''   )
													# This check we did call the fake head request
													mock_head.assert_called()

					@require_tokenizers
					def 			snake_case__      (    self   ):
									_lowerCamelCase        =				mock.Mock()
									_lowerCamelCase        =				5_0_0
									_lowerCamelCase        =				{}
									_lowerCamelCase        =				HTTPError
									_lowerCamelCase        =				{}

									# Download this model to make sure it's in the cache.
									_lowerCamelCase        =				GPTaTokenizerFast.from_pretrained('''gpt2'''   )

									# Under the mock environment we get a 500 error when trying to reach the tokenizer.
									with mock.patch('''requests.Session.request'''				,       return_value=lowerCamelCase__   ) as mock_head:
													_lowerCamelCase        =				GPTaTokenizerFast.from_pretrained('''gpt2'''   )
													# This check we did call the fake head request
													mock_head.assert_called()

					def 			snake_case__      (    self   ):







									try:
													_lowerCamelCase        =				tempfile.mktemp()
													with open(lowerCamelCase__				,       '''wb'''   ) as f:
																	http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''				,       lowerCamelCase__   )

													_lowerCamelCase        =				AlbertTokenizer.from_pretrained(lowerCamelCase__   )
									finally:
													os.remove(lowerCamelCase__   )

									# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
									# the current folder and have the right name.
									if os.path.isfile('''tokenizer.json'''   ):
													# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
													return
									try:
													with open('''tokenizer.json'''				,       '''wb'''   ) as f:
																	http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''				,       lowerCamelCase__   )
													_lowerCamelCase        =				AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''   )
													# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
													self.assertEqual(tokenizer.vocab_size				,       1_0_0_0   )
													# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

									finally:
													os.remove('''tokenizer.json'''   )





					def 			snake_case__      (    self   ):
									_lowerCamelCase        =				AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''   )


@is_staging_test
class 							lowerCamelCase_(   unittest.TestCase    ):





					'''simple docstring'''



					lowercase__   :       Optional[int]										= ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']

					@classmethod
					def 			snake_case__      (    cls   ):
									_lowerCamelCase        =				TOKEN
									HfFolder.save_token(lowerCamelCase__   )

					@classmethod
					def 			snake_case__      (    cls   ):







									try:
													delete_repo(token=cls._token				,       repo_id='''test-tokenizer'''   )
									except HTTPError:
													pass

									try:
													delete_repo(token=cls._token				,       repo_id='''valid_org/test-tokenizer-org'''   )
									except HTTPError:
													pass

									try:
													delete_repo(token=cls._token				,       repo_id='''test-dynamic-tokenizer'''   )
									except HTTPError:
													pass

					def 			snake_case__      (    self   ):







									with tempfile.TemporaryDirectory() as tmp_dir:
													_lowerCamelCase        =				os.path.join(lowerCamelCase__				,       '''vocab.txt'''   )
													with open(lowerCamelCase__				,       '''w'''				,       encoding='''utf-8'''   ) as vocab_writer:
																	vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
													_lowerCamelCase        =				BertTokenizer(lowerCamelCase__   )

									tokenizer.push_to_hub('''test-tokenizer'''				,       use_auth_token=self._token   )
									_lowerCamelCase        =				BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""   )
									self.assertDictEqual(new_tokenizer.vocab				,       tokenizer.vocab   )

									# Reset repo
									delete_repo(token=self._token				,       repo_id='''test-tokenizer'''   )

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(lowerCamelCase__				,       repo_id='''test-tokenizer'''				,       push_to_hub=lowerCamelCase__				,       use_auth_token=self._token   )

									_lowerCamelCase        =				BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""   )
									self.assertDictEqual(new_tokenizer.vocab				,       tokenizer.vocab   )

					def 			snake_case__      (    self   ):







									with tempfile.TemporaryDirectory() as tmp_dir:
													_lowerCamelCase        =				os.path.join(lowerCamelCase__				,       '''vocab.txt'''   )
													with open(lowerCamelCase__				,       '''w'''				,       encoding='''utf-8'''   ) as vocab_writer:
																	vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
													_lowerCamelCase        =				BertTokenizer(lowerCamelCase__   )

									tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''				,       use_auth_token=self._token   )
									_lowerCamelCase        =				BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''   )
									self.assertDictEqual(new_tokenizer.vocab				,       tokenizer.vocab   )

									# Reset repo
									delete_repo(token=self._token				,       repo_id='''valid_org/test-tokenizer-org'''   )

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(
													    lowerCamelCase__				,       repo_id='''valid_org/test-tokenizer-org'''				,       push_to_hub=lowerCamelCase__				,       use_auth_token=self._token   )

									_lowerCamelCase        =				BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''   )
									self.assertDictEqual(new_tokenizer.vocab				,       tokenizer.vocab   )





					@require_tokenizers
					def 			snake_case__      (    self   ):
									CustomTokenizer.register_for_auto_class()
									with tempfile.TemporaryDirectory() as tmp_dir:
													_lowerCamelCase        =				os.path.join(lowerCamelCase__				,       '''vocab.txt'''   )
													with open(lowerCamelCase__				,       '''w'''				,       encoding='''utf-8'''   ) as vocab_writer:
																	vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
													_lowerCamelCase        =				CustomTokenizer(lowerCamelCase__   )

									# No fast custom tokenizer
									tokenizer.push_to_hub('''test-dynamic-tokenizer'''				,       use_auth_token=self._token   )

									_lowerCamelCase        =				AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""				,       trust_remote_code=lowerCamelCase__   )
									# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__				,       '''CustomTokenizer'''   )

									# Fast and slow custom tokenizer
									CustomTokenizerFast.register_for_auto_class()
									with tempfile.TemporaryDirectory() as tmp_dir:
													_lowerCamelCase        =				os.path.join(lowerCamelCase__				,       '''vocab.txt'''   )
													with open(lowerCamelCase__				,       '''w'''				,       encoding='''utf-8'''   ) as vocab_writer:
																	vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )

													_lowerCamelCase        =				BertTokenizerFast.from_pretrained(lowerCamelCase__   )
													bert_tokenizer.save_pretrained(lowerCamelCase__   )
													_lowerCamelCase        =				CustomTokenizerFast.from_pretrained(lowerCamelCase__   )

									tokenizer.push_to_hub('''test-dynamic-tokenizer'''				,       use_auth_token=self._token   )

									_lowerCamelCase        =				AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""				,       trust_remote_code=lowerCamelCase__   )
									# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__				,       '''CustomTokenizerFast'''   )
									_lowerCamelCase        =				AutoTokenizer.from_pretrained(
									    F"""{USER}/test-dynamic-tokenizer"""				,       use_fast=lowerCamelCase__				,       trust_remote_code=lowerCamelCase__   )
									# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__				,       '''CustomTokenizer'''   )


class 							lowerCamelCase_(   unittest.TestCase    ):





					'''simple docstring'''



					def 			snake_case__      (    self   ):
									_lowerCamelCase        =				Trie()
									trie.add('''Hello 友達'''   )
									self.assertEqual(trie.data				,       {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}   )
									trie.add('''Hello'''   )
									trie.data
									self.assertEqual(trie.data				,       {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}   )

					def 			snake_case__      (    self   ):
									_lowerCamelCase        =				Trie()
									self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''   )				,       ['''[CLS] This is a extra_id_100''']   )
									trie.add('''[CLS]'''   )
									trie.add('''extra_id_1'''   )
									trie.add('''extra_id_100'''   )
									self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''   )				,       ['''[CLS]''', ''' This is a ''', '''extra_id_100''']   )

					def 			snake_case__      (    self   ):
									_lowerCamelCase        =				Trie()
									trie.add('''A'''   )
									self.assertEqual(trie.split('''ABC'''   )				,       ['''A''', '''BC''']   )
									self.assertEqual(trie.split('''BCA'''   )				,       ['''BC''', '''A''']   )

					def 			snake_case__      (    self   ):
									_lowerCamelCase        =				Trie()
									trie.add('''TOKEN]'''   )
									trie.add('''[SPECIAL_TOKEN]'''   )
									self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''   )				,       ['''This is something ''', '''[SPECIAL_TOKEN]''']   )

					def 			snake_case__      (    self   ):
									_lowerCamelCase        =				Trie()
									trie.add('''A'''   )
									trie.add('''P'''   )
									trie.add('''[SPECIAL_TOKEN]'''   )
									self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''   )				,       ['''This is something ''', '''[SPECIAL_TOKEN]''']   )

					def 			snake_case__      (    self   ):
									_lowerCamelCase        =				Trie()
									trie.add('''AB'''   )
									trie.add('''B'''   )
									trie.add('''C'''   )
									self.assertEqual(trie.split('''ABC'''   )				,       ['''AB''', '''C''']   )

					def 			snake_case__      (    self   ):
									_lowerCamelCase        =				Trie()
									trie.add('''ABC'''   )
									trie.add('''B'''   )
									trie.add('''CD'''   )
									self.assertEqual(trie.split('''ABCD'''   )				,       ['''ABC''', '''D''']   )





					def 			snake_case__      (    self   ):
									_lowerCamelCase        =				Trie()
									_lowerCamelCase        =				trie.cut_text('''ABC'''				,       [0, 0, 2, 1, 2, 3]   )
									self.assertEqual(lowerCamelCase__				,       ['''AB''', '''C''']   )





"""simple docstring"""
from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
			if not (is_transformers_available() and is_torch_available()):
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			from ...utils.dummy_torch_and_transformers_objects import KandinskyPipeline, KandinskyPriorPipeline
else:
			from .pipeline_kandinsky import KandinskyPipeline
			from .pipeline_kandinsky_imgaimg import KandinskyImgaImgPipeline
			from .pipeline_kandinsky_inpaint import KandinskyInpaintPipeline
			from .pipeline_kandinsky_prior import KandinskyPriorPipeline, KandinskyPriorPipelineOutput
			from .text_encoder import MultilingualCLIP
