
import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_SCREAMING_SNAKE_CASE	       =      get_tests_dir("""fixtures/spiece.model""")







@require_sentencepiece
@require_tokenizers
class 							__magic_name__     (		lowercase__      ,		unittest.TestCase   ):
					_SCREAMING_SNAKE_CASE       :					Optional[Any]     							=   DebertaVaTokenizer
					_SCREAMING_SNAKE_CASE       :					Tuple     							=   DebertaVaTokenizerFast
					_SCREAMING_SNAKE_CASE       :					Union[str, Any]     							=   True
					_SCREAMING_SNAKE_CASE       :					int     							=   True





					def 							lowerCAmelCase      (				self   :							Tuple    ):
												super().setUp()

												# We have a SentencePiece fixture for testing
												__snake_case     			=       DebertaVaTokenizer(snake_case_							,						unk_token="<unk>"    )
												tokenizer.save_pretrained(self.tmpdirname    )





					def 							lowerCAmelCase      (				self   :							Union[str, Any]							,						snake_case_   :							Optional[Any]    ):
												__snake_case     			=       "this is a test"
												__snake_case     			=       "this is a test"
												return input_text, output_text





					def 							lowerCAmelCase      (				self   :							Tuple    ):
												__snake_case     			=       "<pad>"
												__snake_case     			=       0

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case_    )							,						snake_case_    )
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case_    )							,						snake_case_    )





					def 							lowerCAmelCase      (				self   :							Any    ):
												__snake_case     			=       list(self.get_tokenizer().get_vocab().keys()    )
												self.assertEqual(vocab_keys[0]							,						"<pad>"    )
												self.assertEqual(vocab_keys[1]							,						"<unk>"    )
												self.assertEqual(vocab_keys[-1]							,						"[PAD]"    )
												self.assertEqual(len(snake_case_    )							,						30001    )





					def 							lowerCAmelCase      (				self   :							Any    ):
												self.assertEqual(self.get_tokenizer().vocab_size							,						30000    )





					def 							lowerCAmelCase      (				self   :							Dict    ):
												# fmt: off
												__snake_case     			=       " \tHeLLo!how  \n Are yoU?  "
												__snake_case     			=       ["▁hello", "!", "how", "▁are", "▁you", "?"]
												# fmt: on

												__snake_case     			=       DebertaVaTokenizer(snake_case_							,						do_lower_case=snake_case_    )
												__snake_case     			=       tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )

												self.assertListEqual(snake_case_							,						snake_case_    )

												__snake_case     			=       DebertaVaTokenizerFast(snake_case_							,						do_lower_case=snake_case_    )
												__snake_case     			=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )

												self.assertListEqual(snake_case_							,						snake_case_    )





					@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."    )
					def 							lowerCAmelCase      (				self   :							int    ):
												pass





					@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."    )
					def 							lowerCAmelCase      (				self   :							str    ):
												pass





					def 							lowerCAmelCase      (				self   :							int    ):
												# fmt: off
												__snake_case     			=       "I was born in 92000, and this is falsé."
												__snake_case     			=       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
												# fmt: on

												__snake_case     			=       DebertaVaTokenizer(snake_case_							,						split_by_punct=snake_case_    )
												__snake_case     			=       tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )

												self.assertListEqual(snake_case_							,						snake_case_    )

												__snake_case     			=       DebertaVaTokenizerFast(snake_case_							,						split_by_punct=snake_case_    )
												__snake_case     			=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )

												self.assertListEqual(snake_case_							,						snake_case_    )





					def 							lowerCAmelCase      (				self   :							Any    ):
												# fmt: off
												__snake_case     			=       "I was born in 92000, and this is falsé."
												__snake_case     			=       ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
												# fmt: on

												__snake_case     			=       DebertaVaTokenizer(snake_case_							,						do_lower_case=snake_case_							,						split_by_punct=snake_case_    )
												__snake_case     			=       tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )
												self.assertListEqual(snake_case_							,						snake_case_    )

												__snake_case     			=       DebertaVaTokenizerFast(snake_case_							,						do_lower_case=snake_case_							,						split_by_punct=snake_case_    )
												__snake_case     			=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )
												self.assertListEqual(snake_case_							,						snake_case_    )





					def 							lowerCAmelCase      (				self   :							str    ):
												# fmt: off
												__snake_case     			=       "I was born in 92000, and this is falsé."
												__snake_case     			=       ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
												# fmt: on

												__snake_case     			=       DebertaVaTokenizer(snake_case_							,						do_lower_case=snake_case_							,						split_by_punct=snake_case_    )
												__snake_case     			=       tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )

												self.assertListEqual(snake_case_							,						snake_case_    )

												__snake_case     			=       DebertaVaTokenizerFast(snake_case_							,						do_lower_case=snake_case_							,						split_by_punct=snake_case_    )
												__snake_case     			=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )

												self.assertListEqual(snake_case_							,						snake_case_    )





					def 							lowerCAmelCase      (				self   :							Tuple    ):
												# fmt: off
												__snake_case     			=       "I was born in 92000, and this is falsé."
												__snake_case     			=       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
												# fmt: on

												__snake_case     			=       DebertaVaTokenizer(snake_case_							,						do_lower_case=snake_case_							,						split_by_punct=snake_case_    )
												__snake_case     			=       tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )

												self.assertListEqual(snake_case_							,						snake_case_    )

												__snake_case     			=       DebertaVaTokenizerFast(snake_case_							,						do_lower_case=snake_case_							,						split_by_punct=snake_case_    )
												__snake_case     			=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )

												self.assertListEqual(snake_case_							,						snake_case_    )





					def 							lowerCAmelCase      (				self   :							Union[str, Any]    ):
												# fmt: off
												__snake_case     			=       " \tHeLLo!how  \n Are yoU?  "
												__snake_case     			=       ["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
												# fmt: on

												__snake_case     			=       DebertaVaTokenizer(snake_case_							,						do_lower_case=snake_case_							,						split_by_punct=snake_case_    )
												__snake_case     			=       tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )

												self.assertListEqual(snake_case_							,						snake_case_    )

												__snake_case     			=       DebertaVaTokenizerFast(snake_case_							,						do_lower_case=snake_case_							,						split_by_punct=snake_case_    )
												__snake_case     			=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )

												self.assertListEqual(snake_case_							,						snake_case_    )





					def 							lowerCAmelCase      (				self   :							List[str]    ):
												__snake_case     			=       self.get_tokenizer()
												__snake_case     			=       self.get_rust_tokenizer()

												__snake_case     			=       "I was born in 92000, and this is falsé."

												__snake_case     			=       tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )
												__snake_case     			=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )    )
												self.assertListEqual(snake_case_							,						snake_case_    )

												__snake_case     			=       tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )
												__snake_case     			=       rust_tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )

												__snake_case     			=       self.get_rust_tokenizer()
												__snake_case     			=       tokenizer.encode(snake_case_    )
												__snake_case     			=       rust_tokenizer.encode(snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )





					def 							lowerCAmelCase      (				self   :							List[Any]    ):
												__snake_case     			=       "This is a test"
												__snake_case     			=       [13, 1, 4398, 25, 21, 1289]
												__snake_case     			=       ["▁", "T", "his", "▁is", "▁a", "▁test"]
												__snake_case     			=       ["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

												__snake_case     			=       DebertaVaTokenizer(snake_case_							,						keep_accents=snake_case_    )
												__snake_case     			=       DebertaVaTokenizerFast(snake_case_							,						keep_accents=snake_case_    )

												__snake_case     			=       tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )
												__snake_case     			=       tokenizer.tokenize(snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )
												__snake_case     			=       tokenizer.convert_ids_to_tokens(snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )

												__snake_case     			=       rust_tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )
												__snake_case     			=       rust_tokenizer.tokenize(snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )
												__snake_case     			=       rust_tokenizer.convert_ids_to_tokens(snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )

												# fmt: off
												__snake_case     			=       "I was born in 92000, and this is falsé."
												__snake_case     			=       [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
												__snake_case     			=       ["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
												__snake_case     			=       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
												# fmt: on

												__snake_case     			=       tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )
												__snake_case     			=       tokenizer.tokenize(snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )
												__snake_case     			=       tokenizer.convert_ids_to_tokens(snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )

												__snake_case     			=       rust_tokenizer.encode(snake_case_							,						add_special_tokens=snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )
												__snake_case     			=       rust_tokenizer.tokenize(snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )
												__snake_case     			=       rust_tokenizer.convert_ids_to_tokens(snake_case_    )
												self.assertListEqual(snake_case_							,						snake_case_    )





					def 							lowerCAmelCase      (				self   :							Dict    ):
												__snake_case     			=       DebertaVaTokenizer(snake_case_    )

												__snake_case     			=       tokenizer.encode("sequence builders"    )
												__snake_case     			=       tokenizer.encode("multi-sequence build"    )

												__snake_case     			=       tokenizer.build_inputs_with_special_tokens(snake_case_    )
												__snake_case     			=       tokenizer.build_inputs_with_special_tokens(snake_case_							,						snake_case_    )

												self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]							,						snake_case_    )
												self.assertEqual(
												    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]							,						snake_case_							,						)





					@slow
					def 							lowerCAmelCase      (				self   :							Dict    ):
												# fmt: off
												__snake_case     			=       {"input_ids": [[1, 39867, 36, 19390, 486, 27, 35052, 81436, 18, 60685, 1225, 7, 35052, 81436, 18, 9367, 16899, 18, 15937, 53, 594, 773, 18, 16287, 30465, 36, 15937, 6, 41139, 38, 36979, 60763, 191, 6, 34132, 99, 6, 50538, 390, 43230, 6, 34132, 2779, 20850, 14, 699, 1072, 1194, 36, 382, 10901, 53, 7, 699, 1072, 2084, 36, 20422, 630, 53, 19, 105, 3049, 1896, 1053, 16899, 1506, 11, 37978, 4243, 7, 1237, 31869, 200, 16566, 654, 6, 35052, 81436, 7, 55630, 13593, 4, 2], [1, 26, 15011, 13, 667, 8, 1053, 18, 23611, 1237, 72356, 12820, 34, 104134, 1209, 35, 13313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 15785, 14951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=snake_case_							,						model_name="microsoft/deberta-v2-xlarge"							,						revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"							,						)


"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


_SCREAMING_SNAKE_CASE	       =      {
    """configuration_groupvit""": [
        """GROUPVIT_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """GroupViTConfig""",
        """GroupViTOnnxConfig""",
        """GroupViTTextConfig""",
        """GroupViTVisionConfig""",
    ],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_SCREAMING_SNAKE_CASE	       =      [
		    """GROUPVIT_PRETRAINED_MODEL_ARCHIVE_LIST""",
		    """GroupViTModel""",
		    """GroupViTPreTrainedModel""",
		    """GroupViTTextModel""",
		    """GroupViTVisionModel""",
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_SCREAMING_SNAKE_CASE	       =      [
		    """TF_GROUPVIT_PRETRAINED_MODEL_ARCHIVE_LIST""",
		    """TFGroupViTModel""",
		    """TFGroupViTPreTrainedModel""",
		    """TFGroupViTTextModel""",
		    """TFGroupViTVisionModel""",
		]

if TYPE_CHECKING:
		from .configuration_groupvit import (
		    GROUPVIT_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    GroupViTConfig,
		    GroupViTOnnxConfig,
		    GroupViTTextConfig,
		    GroupViTVisionConfig,
		)

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_groupvit import (
				    GROUPVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    GroupViTModel,
				    GroupViTPreTrainedModel,
				    GroupViTTextModel,
				    GroupViTVisionModel,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_groupvit import (
				    TF_GROUPVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFGroupViTModel,
				    TFGroupViTPreTrainedModel,
				    TFGroupViTTextModel,
				    TFGroupViTVisionModel,
				)

else:
		import sys

		_SCREAMING_SNAKE_CASE	       =      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
