

import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase__								=		get_tests_dir("""fixtures/spiece.model""")







@require_sentencepiece
@require_tokenizers
class 	__A    (				_lowercase    ,							unittest.TestCase							):


		"""simple docstring"""




		a__					       =  DebertaVaTokenizer
		a__					       =  DebertaVaTokenizerFast
		a__					       =  True
		a__					       =  True


		def 		A__					(				self):
			super().setUp()

			# We have a SentencePiece fixture for testing
			_UpperCamelCase   :		List[str]			=						DebertaVaTokenizer(__snake_case	,       unk_token='<unk>')
			tokenizer.save_pretrained(self.tmpdirname)


		def 		A__					(				self	,       __snake_case):
			_UpperCamelCase   :		List[Any]			=						'this is a test'
			_UpperCamelCase   :		int			=						'this is a test'
			return input_text, output_text


		def 		A__					(				self):
			_UpperCamelCase   :		List[str]			=						'<pad>'
			_UpperCamelCase   :		Union[str, Any]			=						0

			self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case)	,       __snake_case)
			self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case)	,       __snake_case)


		def 		A__					(				self):
			_UpperCamelCase   :		Tuple			=						list(self.get_tokenizer().get_vocab().keys())
			self.assertEqual(vocab_keys[0]	,       '<pad>')
			self.assertEqual(vocab_keys[1]	,       '<unk>')
			self.assertEqual(vocab_keys[-1]	,       '[PAD]')
			self.assertEqual(len(__snake_case)	,       3_00_01)


		def 		A__					(				self):
			self.assertEqual(self.get_tokenizer().vocab_size	,       3_00_00)


		def 		A__					(				self):
			# fmt: off
			_UpperCamelCase   :		Union[str, Any]			=						' \tHeLLo!how  \n Are yoU?  '
			_UpperCamelCase   :		Tuple			=						['▁hello', '!', 'how', '▁are', '▁you', '?']
			# fmt: on

			_UpperCamelCase   :		str			=						DebertaVaTokenizer(__snake_case	,       do_lower_case=__snake_case)
			_UpperCamelCase   :		Dict			=						tokenizer.convert_ids_to_tokens(tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))

			self.assertListEqual(__snake_case	,       __snake_case)

			_UpperCamelCase   :		int			=						DebertaVaTokenizerFast(__snake_case	,       do_lower_case=__snake_case)
			_UpperCamelCase   :		str			=						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))

			self.assertListEqual(__snake_case	,       __snake_case)


		@unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.')
		def 		A__					(				self):
			pass


		@unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.')
		def 		A__					(				self):
			pass


		def 		A__					(				self):
			# fmt: off
			_UpperCamelCase   :		List[str]			=						'I was born in 92000, and this is falsé.'
			_UpperCamelCase   :		str			=						['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
			# fmt: on

			_UpperCamelCase   :		int			=						DebertaVaTokenizer(__snake_case	,       split_by_punct=__snake_case)
			_UpperCamelCase   :		str			=						tokenizer.convert_ids_to_tokens(tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))

			self.assertListEqual(__snake_case	,       __snake_case)

			_UpperCamelCase   :		Optional[int]			=						DebertaVaTokenizerFast(__snake_case	,       split_by_punct=__snake_case)
			_UpperCamelCase   :		List[Any]			=						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))

			self.assertListEqual(__snake_case	,       __snake_case)


		def 		A__					(				self):
			# fmt: off
			_UpperCamelCase   :		Optional[Any]			=						'I was born in 92000, and this is falsé.'
			_UpperCamelCase   :		List[Any]			=						['▁i', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
			# fmt: on

			_UpperCamelCase   :		Optional[Any]			=						DebertaVaTokenizer(__snake_case	,       do_lower_case=__snake_case	,       split_by_punct=__snake_case)
			_UpperCamelCase   :		List[Any]			=						tokenizer.convert_ids_to_tokens(tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))
			self.assertListEqual(__snake_case	,       __snake_case)

			_UpperCamelCase   :		Optional[Any]			=						DebertaVaTokenizerFast(__snake_case	,       do_lower_case=__snake_case	,       split_by_punct=__snake_case)
			_UpperCamelCase   :		str			=						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))
			self.assertListEqual(__snake_case	,       __snake_case)


		def 		A__					(				self):
			# fmt: off
			_UpperCamelCase   :		Tuple			=						'I was born in 92000, and this is falsé.'
			_UpperCamelCase   :		List[Any]			=						['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
			# fmt: on

			_UpperCamelCase   :		Any			=						DebertaVaTokenizer(__snake_case	,       do_lower_case=__snake_case	,       split_by_punct=__snake_case)
			_UpperCamelCase   :		Tuple			=						tokenizer.convert_ids_to_tokens(tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))

			self.assertListEqual(__snake_case	,       __snake_case)

			_UpperCamelCase   :		Optional[Any]			=						DebertaVaTokenizerFast(__snake_case	,       do_lower_case=__snake_case	,       split_by_punct=__snake_case)
			_UpperCamelCase   :		str			=						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))

			self.assertListEqual(__snake_case	,       __snake_case)


		def 		A__					(				self):
			# fmt: off
			_UpperCamelCase   :		Tuple			=						'I was born in 92000, and this is falsé.'
			_UpperCamelCase   :		int			=						['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
			# fmt: on

			_UpperCamelCase   :		List[str]			=						DebertaVaTokenizer(__snake_case	,       do_lower_case=__snake_case	,       split_by_punct=__snake_case)
			_UpperCamelCase   :		Optional[Any]			=						tokenizer.convert_ids_to_tokens(tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))

			self.assertListEqual(__snake_case	,       __snake_case)

			_UpperCamelCase   :		List[str]			=						DebertaVaTokenizerFast(__snake_case	,       do_lower_case=__snake_case	,       split_by_punct=__snake_case)
			_UpperCamelCase   :		Dict			=						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))

			self.assertListEqual(__snake_case	,       __snake_case)


		def 		A__					(				self):
			# fmt: off
			_UpperCamelCase   :		Optional[Any]			=						' \tHeLLo!how  \n Are yoU?  '
			_UpperCamelCase   :		Dict			=						['▁', '<unk>', 'e', '<unk>', 'o', '!', 'how', '▁', '<unk>', 're', '▁yo', '<unk>', '?']
			# fmt: on

			_UpperCamelCase   :		List[Any]			=						DebertaVaTokenizer(__snake_case	,       do_lower_case=__snake_case	,       split_by_punct=__snake_case)
			_UpperCamelCase   :		List[str]			=						tokenizer.convert_ids_to_tokens(tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))

			self.assertListEqual(__snake_case	,       __snake_case)

			_UpperCamelCase   :		Union[str, Any]			=						DebertaVaTokenizerFast(__snake_case	,       do_lower_case=__snake_case	,       split_by_punct=__snake_case)
			_UpperCamelCase   :		Tuple			=						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))

			self.assertListEqual(__snake_case	,       __snake_case)


		def 		A__					(				self):
			_UpperCamelCase   :		Any			=						self.get_tokenizer()
			_UpperCamelCase   :		Union[str, Any]			=						self.get_rust_tokenizer()

			_UpperCamelCase   :		int			=						'I was born in 92000, and this is falsé.'

			_UpperCamelCase   :		Dict			=						tokenizer.convert_ids_to_tokens(tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))
			_UpperCamelCase   :		Tuple			=						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case))
			self.assertListEqual(__snake_case	,       __snake_case)

			_UpperCamelCase   :		List[str]			=						tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case)
			_UpperCamelCase   :		Optional[int]			=						rust_tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)

			_UpperCamelCase   :		int			=						self.get_rust_tokenizer()
			_UpperCamelCase   :		List[str]			=						tokenizer.encode(__snake_case)
			_UpperCamelCase   :		List[str]			=						rust_tokenizer.encode(__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)


		def 		A__					(				self):
			_UpperCamelCase   :		Dict			=						'This is a test'
			_UpperCamelCase   :		Tuple			=						[13, 1, 43_98, 25, 21, 12_89]
			_UpperCamelCase   :		Any			=						['▁', 'T', 'his', '▁is', '▁a', '▁test']
			_UpperCamelCase   :		Any			=						['▁', '<unk>', 'his', '▁is', '▁a', '▁test']

			_UpperCamelCase   :		Optional[Any]			=						DebertaVaTokenizer(__snake_case	,       keep_accents=__snake_case)
			_UpperCamelCase   :		Union[str, Any]			=						DebertaVaTokenizerFast(__snake_case	,       keep_accents=__snake_case)

			_UpperCamelCase   :		Union[str, Any]			=						tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)
			_UpperCamelCase   :		List[Any]			=						tokenizer.tokenize(__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)
			_UpperCamelCase   :		int			=						tokenizer.convert_ids_to_tokens(__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)

			_UpperCamelCase   :		Dict			=						rust_tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)
			_UpperCamelCase   :		Dict			=						rust_tokenizer.tokenize(__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)
			_UpperCamelCase   :		Union[str, Any]			=						rust_tokenizer.convert_ids_to_tokens(__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)

			# fmt: off
			_UpperCamelCase   :		Optional[Any]			=						'I was born in 92000, and this is falsé.'
			_UpperCamelCase   :		List[str]			=						[13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
			_UpperCamelCase   :		List[str]			=						['▁', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.', ]
			_UpperCamelCase   :		Any			=						['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
			# fmt: on

			_UpperCamelCase   :		Any			=						tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)
			_UpperCamelCase   :		str			=						tokenizer.tokenize(__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)
			_UpperCamelCase   :		Tuple			=						tokenizer.convert_ids_to_tokens(__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)

			_UpperCamelCase   :		Any			=						rust_tokenizer.encode(__snake_case	,       add_special_tokens=__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)
			_UpperCamelCase   :		int			=						rust_tokenizer.tokenize(__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)
			_UpperCamelCase   :		int			=						rust_tokenizer.convert_ids_to_tokens(__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)


		def 		A__					(				self):
			_UpperCamelCase   :		Union[str, Any]			=						DebertaVaTokenizer(__snake_case)

			_UpperCamelCase   :		Tuple			=						tokenizer.encode('sequence builders')
			_UpperCamelCase   :		List[Any]			=						tokenizer.encode('multi-sequence build')

			_UpperCamelCase   :		str			=						tokenizer.build_inputs_with_special_tokens(__snake_case)
			_UpperCamelCase   :		List[str]			=						tokenizer.build_inputs_with_special_tokens(__snake_case	,       __snake_case)

			self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]	,       __snake_case)
			self.assertEqual(
			    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]	,       __snake_case	,       )






		@slow
		def 		A__					(				self):
			# fmt: off
			_UpperCamelCase   :		Optional[Any]			=						{'input_ids': [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
			# fmt: on

			self.tokenizer_integration_test_util(
			    expected_encoding=__snake_case	,       model_name='microsoft/deberta-v2-xlarge'	,       revision='ad6e42c1532ddf3a15c39246b63f5559d558b670'	,       )







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


lowerCAmelCase__								=		{
    """configuration_canine""": ["""CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CanineConfig"""],
    """tokenization_canine""": ["""CanineTokenizer"""],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase__								=		[
							    """CANINE_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """CanineForMultipleChoice""",
							    """CanineForQuestionAnswering""",
							    """CanineForSequenceClassification""",
							    """CanineForTokenClassification""",
							    """CanineLayer""",
							    """CanineModel""",
							    """CaninePreTrainedModel""",
							    """load_tf_weights_in_canine""",
							]


if TYPE_CHECKING:
							from .configuration_canine import CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP, CanineConfig
							from .tokenization_canine import CanineTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_canine import (
														    CANINE_PRETRAINED_MODEL_ARCHIVE_LIST,
														    CanineForMultipleChoice,
														    CanineForQuestionAnswering,
														    CanineForSequenceClassification,
														    CanineForTokenClassification,
														    CanineLayer,
														    CanineModel,
														    CaninePreTrainedModel,
														    load_tf_weights_in_canine,
														)


else:
							import sys

							lowerCAmelCase__								=		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




