"""simple docstring"""




import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_a   :	Optional[Any] 					=     get_tests_dir('fixtures/spiece.model')
@require_sentencepiece
@require_tokenizers
class 		__A							(				SCREAMING_SNAKE_CASE_					,			unittest.TestCase	):
     _UpperCamelCase							:    List[Any]	      =	DebertaVaTokenizer
     _UpperCamelCase							:    Optional[int]	      =	DebertaVaTokenizerFast
     _UpperCamelCase							:    Tuple	      =	True
     _UpperCamelCase							:    List[Any]	      =	True






     def 					__A						(	self						):
       super().setUp()

       # We have a SentencePiece fixture for testing
       _lowerCAmelCase		: List[Any]						    =  DebertaVaTokenizer(a__       ,  unk_token="""<unk>"""						)
       tokenizer.save_pretrained(self.tmpdirname						)






     def 					__A						(	self       ,  a__						):
       _lowerCAmelCase		: List[str]						    =  """this is a test"""
       _lowerCAmelCase		: Any						    =  """this is a test"""
       return input_text, output_text






     def 					__A						(	self						):
       _lowerCAmelCase		: List[str]						    =  """<pad>"""
       _lowerCAmelCase		: Union[str, Any]						    =  0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__						)       ,  a__						)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__						)       ,  a__						)






     def 					__A						(	self						):
       _lowerCAmelCase		: str						    =  list(self.get_tokenizer().get_vocab().keys()						)
       self.assertEqual(vocab_keys[0]       ,  """<pad>"""						)
       self.assertEqual(vocab_keys[1]       ,  """<unk>"""						)
       self.assertEqual(vocab_keys[-1]       ,  """[PAD]"""						)
       self.assertEqual(len(a__						)       ,  30001						)






     def 					__A						(	self						):
       self.assertEqual(self.get_tokenizer().vocab_size       ,  30000						)






     def 					__A						(	self						):
       # fmt: off
       _lowerCAmelCase		: int						    =  """ \tHeLLo!how  \n Are yoU?  """
       _lowerCAmelCase		: Optional[Any]						    =  ["""▁hello""", """!""", """how""", """▁are""", """▁you""", """?"""]
       # fmt: on

       _lowerCAmelCase		: List[Any]						    =  DebertaVaTokenizer(a__       ,  do_lower_case=a__						)
       _lowerCAmelCase		: Tuple						    =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a__       ,  add_special_tokens=a__						)						)

       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: Optional[int]						    =  DebertaVaTokenizerFast(a__       ,  do_lower_case=a__						)
       _lowerCAmelCase		: List[str]						    =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__       ,  add_special_tokens=a__						)						)

       self.assertListEqual(a__       ,  a__						)






     @unittest.skip("""There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."""						)
     def 					__A						(	self						):
       pass






     @unittest.skip("""There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."""						)
     def 					__A						(	self						):
       pass






     def 					__A						(	self						):
       # fmt: off
       _lowerCAmelCase		: Any						    =  """I was born in 92000, and this is falsé."""
       _lowerCAmelCase		: Any						    =  ["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
       # fmt: on

       _lowerCAmelCase		: Dict						    =  DebertaVaTokenizer(a__       ,  split_by_punct=a__						)
       _lowerCAmelCase		: int						    =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a__       ,  add_special_tokens=a__						)						)

       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: List[str]						    =  DebertaVaTokenizerFast(a__       ,  split_by_punct=a__						)
       _lowerCAmelCase		: Optional[Any]						    =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__       ,  add_special_tokens=a__						)						)

       self.assertListEqual(a__       ,  a__						)






     def 					__A						(	self						):
       # fmt: off
       _lowerCAmelCase		: int						    =  """I was born in 92000, and this is falsé."""
       _lowerCAmelCase		: List[str]						    =  ["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
       # fmt: on

       _lowerCAmelCase		: Optional[int]						    =  DebertaVaTokenizer(a__       ,  do_lower_case=a__       ,  split_by_punct=a__						)
       _lowerCAmelCase		: Dict						    =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a__       ,  add_special_tokens=a__						)						)
       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: Tuple						    =  DebertaVaTokenizerFast(a__       ,  do_lower_case=a__       ,  split_by_punct=a__						)
       _lowerCAmelCase		: Union[str, Any]						    =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__       ,  add_special_tokens=a__						)						)
       self.assertListEqual(a__       ,  a__						)






     def 					__A						(	self						):
       # fmt: off
       _lowerCAmelCase		: int						    =  """I was born in 92000, and this is falsé."""
       _lowerCAmelCase		: Tuple						    =  ["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """.""", ]
       # fmt: on

       _lowerCAmelCase		: List[Any]						    =  DebertaVaTokenizer(a__       ,  do_lower_case=a__       ,  split_by_punct=a__						)
       _lowerCAmelCase		: Any						    =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a__       ,  add_special_tokens=a__						)						)

       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: Tuple						    =  DebertaVaTokenizerFast(a__       ,  do_lower_case=a__       ,  split_by_punct=a__						)
       _lowerCAmelCase		: int						    =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__       ,  add_special_tokens=a__						)						)

       self.assertListEqual(a__       ,  a__						)






     def 					__A						(	self						):
       # fmt: off
       _lowerCAmelCase		: List[str]						    =  """I was born in 92000, and this is falsé."""
       _lowerCAmelCase		: List[Any]						    =  ["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
       # fmt: on

       _lowerCAmelCase		: Union[str, Any]						    =  DebertaVaTokenizer(a__       ,  do_lower_case=a__       ,  split_by_punct=a__						)
       _lowerCAmelCase		: str						    =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a__       ,  add_special_tokens=a__						)						)

       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: Any						    =  DebertaVaTokenizerFast(a__       ,  do_lower_case=a__       ,  split_by_punct=a__						)
       _lowerCAmelCase		: Dict						    =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__       ,  add_special_tokens=a__						)						)

       self.assertListEqual(a__       ,  a__						)






     def 					__A						(	self						):
       # fmt: off
       _lowerCAmelCase		: Optional[Any]						    =  """ \tHeLLo!how  \n Are yoU?  """
       _lowerCAmelCase		: Optional[int]						    =  ["""▁""", """<unk>""", """e""", """<unk>""", """o""", """!""", """how""", """▁""", """<unk>""", """re""", """▁yo""", """<unk>""", """?"""]
       # fmt: on

       _lowerCAmelCase		: Any						    =  DebertaVaTokenizer(a__       ,  do_lower_case=a__       ,  split_by_punct=a__						)
       _lowerCAmelCase		: List[str]						    =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a__       ,  add_special_tokens=a__						)						)

       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: Dict						    =  DebertaVaTokenizerFast(a__       ,  do_lower_case=a__       ,  split_by_punct=a__						)
       _lowerCAmelCase		: List[str]						    =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__       ,  add_special_tokens=a__						)						)

       self.assertListEqual(a__       ,  a__						)






     def 					__A						(	self						):
       _lowerCAmelCase		: List[Any]						    =  self.get_tokenizer()
       _lowerCAmelCase		: Optional[int]						    =  self.get_rust_tokenizer()

       _lowerCAmelCase		: Any						    =  """I was born in 92000, and this is falsé."""

       _lowerCAmelCase		: Dict						    =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a__       ,  add_special_tokens=a__						)						)
       _lowerCAmelCase		: Optional[Any]						    =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a__       ,  add_special_tokens=a__						)						)
       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: Optional[Any]						    =  tokenizer.encode(a__       ,  add_special_tokens=a__						)
       _lowerCAmelCase		: Optional[Any]						    =  rust_tokenizer.encode(a__       ,  add_special_tokens=a__						)
       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: Tuple						    =  self.get_rust_tokenizer()
       _lowerCAmelCase		: str						    =  tokenizer.encode(a__						)
       _lowerCAmelCase		: List[str]						    =  rust_tokenizer.encode(a__						)
       self.assertListEqual(a__       ,  a__						)






     def 					__A						(	self						):
       _lowerCAmelCase		: int						    =  """This is a test"""
       _lowerCAmelCase		: Optional[int]						    =  [13, 1, 4398, 25, 21, 1289]
       _lowerCAmelCase		: List[str]						    =  ["""▁""", """T""", """his""", """▁is""", """▁a""", """▁test"""]
       _lowerCAmelCase		: int						    =  ["""▁""", """<unk>""", """his""", """▁is""", """▁a""", """▁test"""]

       _lowerCAmelCase		: Union[str, Any]						    =  DebertaVaTokenizer(a__       ,  keep_accents=a__						)
       _lowerCAmelCase		: str						    =  DebertaVaTokenizerFast(a__       ,  keep_accents=a__						)

       _lowerCAmelCase		: Tuple						    =  tokenizer.encode(a__       ,  add_special_tokens=a__						)
       self.assertListEqual(a__       ,  a__						)
       _lowerCAmelCase		: Dict						    =  tokenizer.tokenize(a__						)
       self.assertListEqual(a__       ,  a__						)
       _lowerCAmelCase		: Optional[Any]						    =  tokenizer.convert_ids_to_tokens(a__						)
       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: Optional[Any]						    =  rust_tokenizer.encode(a__       ,  add_special_tokens=a__						)
       self.assertListEqual(a__       ,  a__						)
       _lowerCAmelCase		: Union[str, Any]						    =  rust_tokenizer.tokenize(a__						)
       self.assertListEqual(a__       ,  a__						)
       _lowerCAmelCase		: Union[str, Any]						    =  rust_tokenizer.convert_ids_to_tokens(a__						)
       self.assertListEqual(a__       ,  a__						)

       # fmt: off
       _lowerCAmelCase		: int						    =  """I was born in 92000, and this is falsé."""
       _lowerCAmelCase		: Tuple						    =  [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
       _lowerCAmelCase		: Any						    =  ["""▁""", """I""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """.""", ]
       _lowerCAmelCase		: Optional[Any]						    =  ["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """.""", ]
       # fmt: on

       _lowerCAmelCase		: List[Any]						    =  tokenizer.encode(a__       ,  add_special_tokens=a__						)
       self.assertListEqual(a__       ,  a__						)
       _lowerCAmelCase		: List[str]						    =  tokenizer.tokenize(a__						)
       self.assertListEqual(a__       ,  a__						)
       _lowerCAmelCase		: Union[str, Any]						    =  tokenizer.convert_ids_to_tokens(a__						)
       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: Optional[int]						    =  rust_tokenizer.encode(a__       ,  add_special_tokens=a__						)
       self.assertListEqual(a__       ,  a__						)
       _lowerCAmelCase		: Tuple						    =  rust_tokenizer.tokenize(a__						)
       self.assertListEqual(a__       ,  a__						)
       _lowerCAmelCase		: Optional[Any]						    =  rust_tokenizer.convert_ids_to_tokens(a__						)
       self.assertListEqual(a__       ,  a__						)






     def 					__A						(	self						):
       _lowerCAmelCase		: Any						    =  DebertaVaTokenizer(a__						)

       _lowerCAmelCase		: List[Any]						    =  tokenizer.encode("""sequence builders"""						)
       _lowerCAmelCase		: List[str]						    =  tokenizer.encode("""multi-sequence build"""						)

       _lowerCAmelCase		: str						    =  tokenizer.build_inputs_with_special_tokens(a__						)
       _lowerCAmelCase		: Any						    =  tokenizer.build_inputs_with_special_tokens(a__       ,  a__						)

       self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]       ,  a__						)
       self.assertEqual(
           [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]       ,  a__       ,  )






     @slow
     def 					__A						(	self						):
       # fmt: off
       _lowerCAmelCase		: List[str]						    =  {"""input_ids""": [[1, 39867, 36, 19390, 486, 27, 35052, 81436, 18, 60685, 1225, 7, 35052, 81436, 18, 9367, 16899, 18, 15937, 53, 594, 773, 18, 16287, 30465, 36, 15937, 6, 41139, 38, 36979, 60763, 191, 6, 34132, 99, 6, 50538, 390, 43230, 6, 34132, 2779, 20850, 14, 699, 1072, 1194, 36, 382, 10901, 53, 7, 699, 1072, 2084, 36, 20422, 630, 53, 19, 105, 3049, 1896, 1053, 16899, 1506, 11, 37978, 4243, 7, 1237, 31869, 200, 16566, 654, 6, 35052, 81436, 7, 55630, 13593, 4, 2], [1, 26, 15011, 13, 667, 8, 1053, 18, 23611, 1237, 72356, 12820, 34, 104134, 1209, 35, 13313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 15785, 14951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=a__       ,  model_name="""microsoft/deberta-v2-xlarge"""       ,  revision="""ad6e42c1532ddf3a15c39246b63f5559d558b670"""       ,  )





"""simple docstring"""




from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
     if not (is_transformers_available() and is_torch_available() and is_transformers_version('>=', '4.25.0')):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline
else:
     from .pipeline_unclip import UnCLIPPipeline
     from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline
     from .text_proj import UnCLIPTextProjModel




