


import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin

class 		lowerCAmelCase     ( __a      ,       unittest.TestCase			):

					'''simple docstring'''

					_A						:			Tuple									=      DebertaTokenizer
					_A						:			Optional[Any]									=      True
					_A						:			str									=      DebertaTokenizerFast



					def       lowerCAmelCase				(  self				:	Optional[Any]							)			->  List[str]:


												"""simple docstring"""

												super().setUp()

												# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
												__lowercase      :							int				=     [
												    "l",
												    "o",
												    "w",
												    "e",
												    "r",
												    "s",
												    "t",
												    "i",
												    "d",
												    "n",
												    "\u0120",
												    "\u0120l",
												    "\u0120n",
												    "\u0120lo",
												    "\u0120low",
												    "er",
												    "\u0120lowest",
												    "\u0120newer",
												    "\u0120wider",
												    "[UNK]",
												]
												__lowercase      :							str				=     dict(zip(__A		,     range(len(__A							)							)							)							)
												__lowercase      :							List[Any]				=     ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
												__lowercase      :							List[Any]				=     {"unk_token": "[UNK]"}

												__lowercase      :							List[Any]				=     os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["""vocab_file"""]							)
												__lowercase      :							Any				=     os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["""merges_file"""]							)
												with open(self.vocab_file		,     """w"""		,     encoding="""utf-8"""							) as fp:
																			fp.write(json.dumps(__A							) + """\n"""							)
												with open(self.merges_file		,     """w"""		,     encoding="""utf-8"""							) as fp:
																			fp.write("""\n""".join(__A							)							)



					def       lowerCAmelCase				(  self				:	str		,     **__a				:	Union[str, Any]							)			->  List[Any]:


												"""simple docstring"""

												kwargs.update(self.special_tokens_map							)
												return self.tokenizer_class.from_pretrained(self.tmpdirname		,     **__A							)



					def       lowerCAmelCase				(  self				:	str		,     __a				:	Optional[Any]							)			->  Tuple:


												"""simple docstring"""

												__lowercase      :							Optional[Any]				=     "lower newer"
												__lowercase      :							Any				=     "lower newer"
												return input_text, output_text



					def       lowerCAmelCase				(  self				:	Any							)			->  Any:


												"""simple docstring"""

												__lowercase      :							Any				=     self.get_tokenizer()
												__lowercase      :							Any				=     "lower newer"
												__lowercase      :							Optional[Any]				=     ["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
												__lowercase      :							List[Any]				=     tokenizer.tokenize(__A							)
												self.assertListEqual(__A		,     __A							)

												__lowercase      :							Dict				=     tokens + [tokenizer.unk_token]
												__lowercase      :							List[Any]				=     [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(__A							)		,     __A							)



					def       lowerCAmelCase				(  self				:	int							)			->  Optional[int]:


												"""simple docstring"""

												__lowercase      :							Tuple				=     self.get_tokenizer()
												__lowercase      :							List[Any]				=     tokenizer("""Hello"""		,     """World"""							)
												__lowercase      :							Union[str, Any]				=     [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
												self.assertListEqual(tokd["""token_type_ids"""]		,     __A							)



					@slow
					def       lowerCAmelCase				(  self				:	Optional[int]							)			->  int:


												"""simple docstring"""

												__lowercase      :							Optional[int]				=     self.tokenizer_class.from_pretrained("""microsoft/deberta-base"""							)

												__lowercase      :							int				=     tokenizer.encode("""sequence builders"""		,     add_special_tokens=__A							)
												__lowercase      :							int				=     tokenizer.encode("""multi-sequence build"""		,     add_special_tokens=__A							)

												__lowercase      :							Any				=     tokenizer.encode(
												    """sequence builders"""		,     add_special_tokens=__A		,     add_prefix_space=__A							)
												__lowercase      :							Optional[int]				=     tokenizer.encode(
												    """sequence builders"""		,     """multi-sequence build"""		,     add_special_tokens=__A		,     add_prefix_space=__A							)

												__lowercase      :							str				=     tokenizer.build_inputs_with_special_tokens(__A							)
												__lowercase      :							Tuple				=     tokenizer.build_inputs_with_special_tokens(__A		,     __A							)

												assert encoded_sentence == encoded_text_from_decode
												assert encoded_pair == encoded_pair_from_decode



					@slow
					def       lowerCAmelCase				(  self				:	str							)			->  Dict:


												"""simple docstring"""

												__lowercase      :							Optional[int]				=     [self.tokenizer_class]
												if self.test_rust_tokenizer:
																			tokenizer_classes.append(self.rust_tokenizer_class							)

												for tokenizer_class in tokenizer_classes:
																			__lowercase      :							List[str]				=     tokenizer_class.from_pretrained("""microsoft/deberta-base"""							)

																			__lowercase      :							str				=     [
																			    "ALBERT: A Lite BERT for Self-supervised Learning of Language Representations",
																			    "ALBERT incorporates two parameter reduction techniques",
																			    "The first one is a factorized embedding parameterization. By decomposing the large vocabulary"
																			    " embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"
																			    " vocabulary embedding.",
																			]

																			__lowercase      :							Dict				=     tokenizer(__A		,     padding=__A							)
																			__lowercase      :							Optional[int]				=     [tokenizer.decode(__A		,     skip_special_tokens=__A							) for seq in encoding["input_ids"]]

																			# fmt: off
																			__lowercase      :							Tuple				=     {
																			    "input_ids": [
																			        [1, 2118, 11126, 565, 35, 83, 25191, 163, 18854, 13, 12156, 12, 16101, 25376, 13807, 9, 22205, 27893, 1635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
																			        [1, 2118, 11126, 565, 24536, 80, 43797, 4878, 7373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
																			        [1, 133, 78, 65, 16, 10, 3724, 1538, 33183, 11303, 43797, 1938, 4, 870, 24165, 29105, 5, 739, 32644, 33183, 11303, 36173, 88, 80, 650, 7821, 45940, 6, 52, 2559, 5, 1836, 9, 5, 7397, 13171, 31, 5, 1836, 9, 32644, 33183, 11303, 4, 2]
																			    ],
																			    "token_type_ids": [
																			        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
																			        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
																			        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
																			    ],
																			    "attention_mask": [
																			        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
																			        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
																			        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
																			    ]
																			}
																			# fmt: on

																			__lowercase      :							str				=     [
																			    "ALBERT: A Lite BERT for Self-supervised Learning of Language Representations",
																			    "ALBERT incorporates two parameter reduction techniques",
																			    "The first one is a factorized embedding parameterization. By decomposing the large vocabulary"
																			    " embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"
																			    " vocabulary embedding.",
																			]

																			self.assertDictEqual(encoding.data		,     __A							)

																			for expected, decoded in zip(__A		,     __A							):
																										self.assertEqual(__A		,     __A							)







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase : int						   =       {
    '''configuration_funnel''': ['''FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''FunnelConfig'''],
    '''convert_funnel_original_tf_checkpoint_to_pytorch''': [],
    '''tokenization_funnel''': ['''FunnelTokenizer'''],
}

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCamelCase : List[str]						   =       ['''FunnelTokenizerFast''']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCamelCase : Union[str, Any]						   =       [
	    '''FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''FunnelBaseModel''',
	    '''FunnelForMaskedLM''',
	    '''FunnelForMultipleChoice''',
	    '''FunnelForPreTraining''',
	    '''FunnelForQuestionAnswering''',
	    '''FunnelForSequenceClassification''',
	    '''FunnelForTokenClassification''',
	    '''FunnelModel''',
	    '''FunnelPreTrainedModel''',
	    '''load_tf_weights_in_funnel''',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCamelCase : str						   =       [
	    '''TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''TFFunnelBaseModel''',
	    '''TFFunnelForMaskedLM''',
	    '''TFFunnelForMultipleChoice''',
	    '''TFFunnelForPreTraining''',
	    '''TFFunnelForQuestionAnswering''',
	    '''TFFunnelForSequenceClassification''',
	    '''TFFunnelForTokenClassification''',
	    '''TFFunnelModel''',
	    '''TFFunnelPreTrainedModel''',
	]


if TYPE_CHECKING:
	from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
	from .tokenization_funnel import FunnelTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_funnel_fast import FunnelTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_funnel import (
		    FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
		    FunnelBaseModel,
		    FunnelForMaskedLM,
		    FunnelForMultipleChoice,
		    FunnelForPreTraining,
		    FunnelForQuestionAnswering,
		    FunnelForSequenceClassification,
		    FunnelForTokenClassification,
		    FunnelModel,
		    FunnelPreTrainedModel,
		    load_tf_weights_in_funnel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_funnel import (
		    TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFFunnelBaseModel,
		    TFFunnelForMaskedLM,
		    TFFunnelForMultipleChoice,
		    TFFunnelForPreTraining,
		    TFFunnelForQuestionAnswering,
		    TFFunnelForSequenceClassification,
		    TFFunnelForTokenClassification,
		    TFFunnelModel,
		    TFFunnelPreTrainedModel,
		)

else:
	import sys

	lowerCamelCase : Any						   =       _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)