

import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase__								=		get_tests_dir("""fixtures/test_sentencepiece_no_bos.model""")







@require_sentencepiece
@require_tokenizers
class 	lowercase    (				a__    ,							unittest.TestCase							):


  """simple docstring"""




  a__					       =  PegasusTokenizer
  a__					       =  PegasusTokenizerFast
  a__					       =  True
  a__					       =  True


  def 		A__					(				self):
   super().setUp()

   # We have a SentencePiece fixture for testing
   _UpperCamelCase   :		str			=						PegasusTokenizer(_A)
   tokenizer.save_pretrained(self.tmpdirname)


  @cached_property
  def 		A__					(				self):
   return PegasusTokenizer.from_pretrained('google/pegasus-large')


  def 		A__					(				self	,       **__snake_case):
   return PegasusTokenizer.from_pretrained(self.tmpdirname	,       **_A)


  def 		A__					(				self	,       __snake_case):
   return ("This is a test", "This is a test")


  def 		A__					(				self):
   _UpperCamelCase   :		Dict			=						'</s>'
   _UpperCamelCase   :		str			=						1

   self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A)	,       _A)
   self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A)	,       _A)


  def 		A__					(				self):
   _UpperCamelCase   :		Optional[Any]			=						list(self.get_tokenizer().get_vocab().keys())

   self.assertEqual(vocab_keys[0]	,       '<pad>')
   self.assertEqual(vocab_keys[1]	,       '</s>')
   self.assertEqual(vocab_keys[-1]	,       'v')
   self.assertEqual(len(_A)	,       11_03)


  def 		A__					(				self):
   self.assertEqual(self.get_tokenizer().vocab_size	,       11_03)


  def 		A__					(				self):
   _UpperCamelCase   :		str			=						self.rust_tokenizer_class.from_pretrained(self.tmpdirname)
   _UpperCamelCase   :		Any			=						self.tokenizer_class.from_pretrained(self.tmpdirname)
   _UpperCamelCase   :		int			=						(
       'Let\'s see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important'
       ' </s> <pad> <pad> <pad>'
   )
   _UpperCamelCase   :		str			=						rust_tokenizer([raw_input_str]	,       return_tensors=_A	,       add_special_tokens=_A).input_ids[0]
   _UpperCamelCase   :		List[Any]			=						py_tokenizer([raw_input_str]	,       return_tensors=_A	,       add_special_tokens=_A).input_ids[0]
   self.assertListEqual(_A	,       _A)


  def 		A__					(				self):
   _UpperCamelCase   :		str			=						self._large_tokenizer
   # <mask_1> masks whole sentence while <mask_2> masks single word
   _UpperCamelCase   :		Optional[Any]			=						'<mask_1> To ensure a <mask_2> flow of bank resolutions.'
   _UpperCamelCase   :		str			=						[2, 4_13, 6_15, 1_14, 3, 19_71, 1_13, 16_79, 1_07_10, 1_07, 1]
   _UpperCamelCase   :		Union[str, Any]			=						tokenizer([raw_input_str]	,       return_tensors=_A).input_ids[0]
   self.assertListEqual(_A	,       _A)


  def 		A__					(				self):
   _UpperCamelCase   :		Tuple			=						self._large_tokenizer
   # The tracebacks for the following asserts are **better** without messages or self.assertEqual
   assert tokenizer.vocab_size == 9_61_03
   assert tokenizer.pad_token_id == 0
   assert tokenizer.eos_token_id == 1
   assert tokenizer.offset == 1_03
   assert tokenizer.unk_token_id == tokenizer.offset + 2 == 1_05
   assert tokenizer.unk_token == "<unk>"
   assert tokenizer.model_max_length == 10_24
   _UpperCamelCase   :		Dict			=						'To ensure a smooth flow of bank resolutions.'
   _UpperCamelCase   :		Dict			=						[4_13, 6_15, 1_14, 22_91, 19_71, 1_13, 16_79, 1_07_10, 1_07, 1]
   _UpperCamelCase   :		str			=						tokenizer([raw_input_str]	,       return_tensors=_A).input_ids[0]
   self.assertListEqual(_A	,       _A)
   assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]


  @require_torch
  def 		A__					(				self):
   _UpperCamelCase   :		List[Any]			=						['This is going to be way too long.' * 1_50, 'short example']
   _UpperCamelCase   :		List[Any]			=						['not super long but more than 5 tokens', 'tiny']
   _UpperCamelCase   :		List[Any]			=						self._large_tokenizer(_A	,       padding=_A	,       truncation=_A	,       return_tensors='pt')
   _UpperCamelCase   :		str			=						self._large_tokenizer(
       text_target=_A	,       max_length=5	,       padding=_A	,       truncation=_A	,       return_tensors='pt')

   assert batch.input_ids.shape == (2, 10_24)
   assert batch.attention_mask.shape == (2, 10_24)
   assert targets["input_ids"].shape == (2, 5)
   assert len(_A) == 2  # input_ids, attention_mask.






  @slow
  def 		A__					(				self):
   _UpperCamelCase   :		Tuple			=						{'input_ids': [[3_89_79, 1_43, 1_84_85, 6_06, 1_30, 2_66_69, 8_76_86, 1_21, 5_41_89, 11_29, 1_11, 2_66_69, 8_76_86, 1_21, 91_14, 1_47_87, 1_21, 1_32_49, 1_58, 5_92, 9_56, 1_21, 1_46_21, 3_15_76, 1_43, 6_26_13, 1_08, 96_88, 9_30, 4_34_30, 1_15_62, 6_26_13, 3_04, 1_08, 1_14_43, 8_97, 1_08, 93_14, 1_74_15, 6_33_99, 1_08, 1_14_43, 76_14, 1_83_16, 1_18, 42_84, 71_48, 1_24_30, 1_43, 14_00, 2_57_03, 1_58, 1_11, 42_84, 71_48, 1_17_72, 1_43, 2_12_97, 10_64, 1_58, 1_22, 2_04, 35_06, 17_54, 11_33, 1_47_87, 15_81, 1_15, 3_32_24, 44_82, 1_11, 13_55, 1_10, 2_91_73, 3_17, 5_08_33, 1_08, 2_01_47, 9_46_65, 1_11, 7_71_98, 1_07, 1], [1_10, 6_26_13, 1_17, 6_38, 1_12, 11_33, 1_21, 2_00_98, 13_55, 7_90_50, 1_38_72, 1_35, 15_96, 5_35_41, 13_52, 1_41, 1_30_39, 55_42, 1_24, 3_02, 5_18, 1_11, 2_68, 29_56, 1_15, 1_49, 44_27, 1_07, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_39, 12_35, 27_99, 1_82_89, 1_77_80, 2_04, 1_09, 94_74, 12_96, 1_07, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
   # fmt: on

   self.tokenizer_integration_test_util(
       expected_encoding=_A	,       model_name='google/bigbird-pegasus-large-arxiv'	,       revision='ba85d0851d708441f91440d509690f1ab6353415'	,       )







@require_sentencepiece
@require_tokenizers
class 	lowercase    (				a__    ,							unittest.TestCase							):


  """simple docstring"""




  a__					       =  PegasusTokenizer
  a__					       =  PegasusTokenizerFast
  a__					       =  True
  a__					       =  True


  def 		A__					(				self):
   super().setUp()

   # We have a SentencePiece fixture for testing
   _UpperCamelCase   :		Optional[Any]			=						PegasusTokenizer(_A	,       offset=0	,       mask_token_sent=_A	,       mask_token='[MASK]')
   tokenizer.save_pretrained(self.tmpdirname)


  @cached_property
  def 		A__					(				self):
   return PegasusTokenizer.from_pretrained('google/bigbird-pegasus-large-arxiv')


  def 		A__					(				self	,       **__snake_case):
   return PegasusTokenizer.from_pretrained(self.tmpdirname	,       **_A)


  def 		A__					(				self	,       __snake_case):
   return ("This is a test", "This is a test")


  def 		A__					(				self):
   _UpperCamelCase   :		str			=						self.rust_tokenizer_class.from_pretrained(self.tmpdirname)
   _UpperCamelCase   :		Optional[Any]			=						self.tokenizer_class.from_pretrained(self.tmpdirname)
   _UpperCamelCase   :		List[str]			=						(
       'Let\'s see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>'
       ' <pad> <pad> <pad>'
   )
   _UpperCamelCase   :		List[str]			=						rust_tokenizer([raw_input_str]	,       return_tensors=_A	,       add_special_tokens=_A).input_ids[0]
   _UpperCamelCase   :		Tuple			=						py_tokenizer([raw_input_str]	,       return_tensors=_A	,       add_special_tokens=_A).input_ids[0]
   self.assertListEqual(_A	,       _A)


  @require_torch
  def 		A__					(				self):
   _UpperCamelCase   :		Optional[Any]			=						['This is going to be way too long.' * 10_00, 'short example']
   _UpperCamelCase   :		Dict			=						['not super long but more than 5 tokens', 'tiny']
   _UpperCamelCase   :		List[Any]			=						self._large_tokenizer(_A	,       padding=_A	,       truncation=_A	,       return_tensors='pt')
   _UpperCamelCase   :		int			=						self._large_tokenizer(
       text_target=_A	,       max_length=5	,       padding=_A	,       truncation=_A	,       return_tensors='pt')

   assert batch.input_ids.shape == (2, 40_96)
   assert batch.attention_mask.shape == (2, 40_96)
   assert targets["input_ids"].shape == (2, 5)
   assert len(_A) == 2  # input_ids, attention_mask.






  def 		A__					(				self):
   _UpperCamelCase   :		Tuple			=						(
       'This is an example string that is used to test the original TF implementation against the HF'
       ' implementation'
   )

   _UpperCamelCase   :		int			=						self._large_tokenizer(_A).input_ids

   self.assertListEqual(
       _A	,       [1_82, 1_17, 1_42, 5_87, 42_11, 1_20, 1_17, 2_63, 1_12, 8_04, 1_09, 8_56, 2_50_16, 31_37, 4_64, 1_09, 2_69_55, 31_37, 1]	,       )







from ...processing_utils import ProcessorMixin







class 	lowercase    (				_lowercase							):


		"""simple docstring"""




		a__					       =  ["image_processor", "feature_extractor"]
		a__					       =  "TvltImageProcessor"
		a__					       =  "TvltFeatureExtractor"


		def __init__(				self	,       __snake_case	,       __snake_case):
			super().__init__(image_processor=__snake_case	,       feature_extractor=__snake_case)

			_UpperCamelCase   :		List[str]			=						image_processor
			_UpperCamelCase   :		Dict			=						feature_extractor


		def __call__(				self	,       __snake_case=None	,       __snake_case=None	,       __snake_case=None	,       __snake_case=None	,       __snake_case=False	,       __snake_case=False	,       *__snake_case	,       **__snake_case	,       ):

			if images is None and audio is None:
				raise ValueError('You need to specify either an `images` or `audio` input to process.')

			_UpperCamelCase   :		Union[str, Any]			=						None
			if images is not None:
				_UpperCamelCase   :		Tuple			=						self.image_processor(__snake_case	,       mask_pixel=__snake_case	,       *__snake_case	,       **__snake_case)
			if images_mixed is not None:
				_UpperCamelCase   :		Union[str, Any]			=						self.image_processor(__snake_case	,       is_mixed=__snake_case	,       *__snake_case	,       **__snake_case)
			if audio is not None:
				_UpperCamelCase   :		Tuple			=						self.feature_extractor(
				    __snake_case	,       *__snake_case	,       sampling_rate=__snake_case	,       mask_audio=__snake_case	,       **__snake_case)

			_UpperCamelCase   :		Tuple			=						{}
			if audio is not None:
				output_dict.update(__snake_case)
			if images is not None:
				output_dict.update(__snake_case)
			if images_mixed_dict is not None:
				output_dict.update(__snake_case)
			return output_dict






		@property
		def 		A__					(				self):
			_UpperCamelCase   :		List[Any]			=						self.image_processor.model_input_names
			_UpperCamelCase   :		List[Any]			=						self.feature_extractor.model_input_names
			return list(dict.fromkeys(image_processor_input_names + feature_extractor_input_names))




