

import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__  :      Optional[Any]			=    get_tests_dir("fixtures/test_sentencepiece_bpe_char.model")



@require_sentencepiece
@require_tokenizers
class       __snake_case  (    lowerCAmelCase__     , unittest.TestCase       ):
  __lowerCAmelCase     :	Tuple						 =  SpeechTaTokenizer
  __lowerCAmelCase     :	int						 =  False
  __lowerCAmelCase     :	int						 =  True

  def 						lowerCAmelCase__     (					self):
      super().setUp()

      # We have a SentencePiece fixture for testing
      SCREAMING_SNAKE_CASE_									=						SpeechTaTokenizer(_A)

      SCREAMING_SNAKE_CASE_									=						AddedToken('<mask>'						,		lstrip=_A						,		rstrip=_A)
      SCREAMING_SNAKE_CASE_									=						mask_token
      tokenizer.add_special_tokens({'mask_token': mask_token})
      tokenizer.add_tokens(['<ctc_blank>'])

      tokenizer.save_pretrained(self.tmpdirname)

  def 						lowerCAmelCase__     (					self						,		_A):
      SCREAMING_SNAKE_CASE_									=						'this is a test'
      SCREAMING_SNAKE_CASE_									=						'this is a test'
      return input_text, output_text

  def 						lowerCAmelCase__     (					self						,		_A						,		_A=False						,		_A=20						,		_A=5):
      SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_									=						self.get_input_output_texts(_A)
      SCREAMING_SNAKE_CASE_									=						tokenizer.encode(_A						,		add_special_tokens=_A)
      SCREAMING_SNAKE_CASE_									=						tokenizer.decode(_A						,		clean_up_tokenization_spaces=_A)
      return text, ids

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						'<pad>'
      SCREAMING_SNAKE_CASE_									=						1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A)						,		_A)
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A)						,		_A)

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						list(self.get_tokenizer().get_vocab().keys())

      self.assertEqual(vocab_keys[0]						,		'<s>')
      self.assertEqual(vocab_keys[1]						,		'<pad>')
      self.assertEqual(vocab_keys[-4]						,		'œ')
      self.assertEqual(vocab_keys[-2]						,		'<mask>')
      self.assertEqual(vocab_keys[-1]						,		'<ctc_blank>')
      self.assertEqual(len(_A)						,		81)

  def 						lowerCAmelCase__     (					self):
      self.assertEqual(self.get_tokenizer().vocab_size						,		79)

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						self.get_tokenizers(do_lower_case=_A)
      for tokenizer in tokenizers:
          with self.subTest(f"""{tokenizer.__class__.__name__}"""):
              SCREAMING_SNAKE_CASE_									=						tokenizer.vocab_size
              SCREAMING_SNAKE_CASE_									=						len(_A)

              self.assertNotEqual(_A						,		0)

              # We usually have added tokens from the start in tests because our vocab fixtures are
              # smaller than the original vocabs - let's not assert this
              # self.assertEqual(vocab_size, all_size)

              SCREAMING_SNAKE_CASE_									=						['aaaaa bbbbbb', 'cccccccccdddddddd']
              SCREAMING_SNAKE_CASE_									=						tokenizer.add_tokens(_A)
              SCREAMING_SNAKE_CASE_									=						tokenizer.vocab_size
              SCREAMING_SNAKE_CASE_									=						len(_A)

              self.assertNotEqual(_A						,		0)
              self.assertEqual(_A						,		_A)
              self.assertEqual(_A						,		len(_A))
              self.assertEqual(_A						,		all_size + len(_A))

              SCREAMING_SNAKE_CASE_									=						tokenizer.encode('aaaaa bbbbbb low cccccccccdddddddd l'						,		add_special_tokens=_A)

              self.assertGreaterEqual(len(_A)						,		4)
              self.assertGreater(tokens[0]						,		tokenizer.vocab_size - 1)
              self.assertGreater(tokens[-3]						,		tokenizer.vocab_size - 1)

              SCREAMING_SNAKE_CASE_									=						{'eos_token': '>>>>|||<||<<|<<', 'pad_token': '<<<<<|||>|>>>>|>'}
              SCREAMING_SNAKE_CASE_									=						tokenizer.add_special_tokens(_A)
              SCREAMING_SNAKE_CASE_									=						tokenizer.vocab_size
              SCREAMING_SNAKE_CASE_									=						len(_A)

              self.assertNotEqual(_A						,		0)
              self.assertEqual(_A						,		_A)
              self.assertEqual(_A						,		len(_A))
              self.assertEqual(_A						,		all_size_a + len(_A))

              SCREAMING_SNAKE_CASE_									=						tokenizer.encode(
                  '>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'						,		add_special_tokens=_A)

              self.assertGreaterEqual(len(_A)						,		6)
              self.assertGreater(tokens[0]						,		tokenizer.vocab_size - 1)
              self.assertGreater(tokens[0]						,		tokens[1])
              self.assertGreater(tokens[-3]						,		tokenizer.vocab_size - 1)
              self.assertGreater(tokens[-3]						,		tokens[-4])
              self.assertEqual(tokens[0]						,		tokenizer.eos_token_id)
              self.assertEqual(tokens[-3]						,		tokenizer.pad_token_id)

  def 						lowerCAmelCase__     (					self):
      pass

  def 						lowerCAmelCase__     (					self):
      pass

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						self.get_tokenizer()

      SCREAMING_SNAKE_CASE_									=						tokenizer.tokenize('This is a test')
      # fmt: off
      self.assertListEqual(_A						,		[SPIECE_UNDERLINE, 'T', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'a', SPIECE_UNDERLINE, 't', 'e', 's', 't'])
      # fmt: on

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(_A)						,		[4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]						,		)

      SCREAMING_SNAKE_CASE_									=						tokenizer.tokenize('I was born in 92000, and this is falsé.')
      self.assertListEqual(
          _A						,		[SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '92000', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.'])

      SCREAMING_SNAKE_CASE_									=						tokenizer.convert_tokens_to_ids(_A)
      # fmt: off
      self.assertListEqual(_A						,		[4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26])
      # fmt: on

      SCREAMING_SNAKE_CASE_									=						tokenizer.convert_ids_to_tokens(_A)
      self.assertListEqual(
          _A						,		[SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '<unk>', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.'])


  @slow
  def 						lowerCAmelCase__     (					self):
      # Use custom sequence because this tokenizer does not handle numbers.
      SCREAMING_SNAKE_CASE_									=						[
          'Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '
          'general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '
          'Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '
          'models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.',
          'BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '
          'conditioning on both left and right context in all layers.',
          'The quick brown fox jumps over the lazy dog.',
      ]

      # fmt: off
      SCREAMING_SNAKE_CASE_									=						{
          'input_ids': [
              [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
              [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
              [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
          ],
          'attention_mask': [
              [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
              [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
              [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
          ]
      }
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=_A						,		model_name='microsoft/speecht5_asr'						,		revision='c5ef64c71905caeccde0e4462ef3f9077224c524'						,		sequences=_A						,		)





import unittest

from transformers import TrOCRConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers.models.trocr.modeling_trocr import TrOCRDecoder, TrOCRForCausalLM



@require_torch
class       __snake_case  :

  def __init__(					self						,		_A						,		_A=99						,		_A=13						,		_A=16						,		_A=7						,		_A=True						,		_A=True						,		_A=True						,		_A=False						,		_A=True						,		_A=2						,		_A=32						,		_A=4						,		_A=4						,		_A=30						,		_A=0						,		_A=1						,		_A=2						,		_A=None						,		):
      SCREAMING_SNAKE_CASE_									=						parent
      SCREAMING_SNAKE_CASE_									=						batch_size
      SCREAMING_SNAKE_CASE_									=						decoder_seq_length
      # For common tests
      SCREAMING_SNAKE_CASE_									=						self.decoder_seq_length
      SCREAMING_SNAKE_CASE_									=						is_training
      SCREAMING_SNAKE_CASE_									=						use_attention_mask
      SCREAMING_SNAKE_CASE_									=						use_labels

      SCREAMING_SNAKE_CASE_									=						vocab_size
      SCREAMING_SNAKE_CASE_									=						d_model
      SCREAMING_SNAKE_CASE_									=						d_model
      SCREAMING_SNAKE_CASE_									=						decoder_layers
      SCREAMING_SNAKE_CASE_									=						decoder_layers
      SCREAMING_SNAKE_CASE_									=						decoder_ffn_dim
      SCREAMING_SNAKE_CASE_									=						decoder_attention_heads
      SCREAMING_SNAKE_CASE_									=						decoder_attention_heads
      SCREAMING_SNAKE_CASE_									=						eos_token_id
      SCREAMING_SNAKE_CASE_									=						bos_token_id
      SCREAMING_SNAKE_CASE_									=						pad_token_id
      SCREAMING_SNAKE_CASE_									=						decoder_start_token_id
      SCREAMING_SNAKE_CASE_									=						use_cache
      SCREAMING_SNAKE_CASE_									=						max_position_embeddings

      SCREAMING_SNAKE_CASE_									=						None
      SCREAMING_SNAKE_CASE_									=						decoder_seq_length
      SCREAMING_SNAKE_CASE_									=						2
      SCREAMING_SNAKE_CASE_									=						1

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						ids_tensor([self.batch_size, self.decoder_seq_length]						,		self.vocab_size)

      SCREAMING_SNAKE_CASE_									=						None
      if self.use_attention_mask:
          SCREAMING_SNAKE_CASE_									=						ids_tensor([self.batch_size, self.decoder_seq_length]						,		vocab_size=2)

      SCREAMING_SNAKE_CASE_									=						None
      if self.use_labels:
          SCREAMING_SNAKE_CASE_									=						ids_tensor([self.batch_size, self.decoder_seq_length]						,		self.vocab_size)

      SCREAMING_SNAKE_CASE_									=						TrOCRConfig(
          vocab_size=self.vocab_size						,		d_model=self.d_model						,		decoder_layers=self.decoder_layers						,		decoder_ffn_dim=self.decoder_ffn_dim						,		decoder_attention_heads=self.decoder_attention_heads						,		eos_token_id=self.eos_token_id						,		bos_token_id=self.bos_token_id						,		use_cache=self.use_cache						,		pad_token_id=self.pad_token_id						,		decoder_start_token_id=self.decoder_start_token_id						,		max_position_embeddings=self.max_position_embeddings						,		)

      return (config, input_ids, attention_mask, lm_labels)

  def 						lowerCAmelCase__     (					self						,		_A						,		_A						,		_A						,		_A						,		):
      SCREAMING_SNAKE_CASE_									=						True
      SCREAMING_SNAKE_CASE_									=						TrOCRDecoder(config=_A).to(_A).eval()
      SCREAMING_SNAKE_CASE_									=						input_ids[:2]

      input_ids[input_ids == 0] += 1
      # first forward pass
      SCREAMING_SNAKE_CASE_									=						model(_A						,		use_cache=_A)
      SCREAMING_SNAKE_CASE_									=						model(_A)
      SCREAMING_SNAKE_CASE_									=						model(_A						,		use_cache=_A)

      self.parent.assertTrue(len(_A) == len(_A))
      self.parent.assertTrue(len(_A) == len(_A) + 1)

      SCREAMING_SNAKE_CASE_									=						outputs['past_key_values']

      # create hypothetical next token and extent to next_input_ids
      SCREAMING_SNAKE_CASE_									=						ids_tensor((2, 1)						,		config.vocab_size - 1) + 1

      # append to next input_ids and
      SCREAMING_SNAKE_CASE_									=						torch.cat([input_ids, next_tokens]						,		dim=-1)

      SCREAMING_SNAKE_CASE_									=						model(_A)['last_hidden_state']
      SCREAMING_SNAKE_CASE_									=						model(_A						,		past_key_values=_A)['last_hidden_state']

      # select random slice
      SCREAMING_SNAKE_CASE_									=						ids_tensor((1,)						,		output_from_past.shape[-1]).item()
      SCREAMING_SNAKE_CASE_									=						output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
      SCREAMING_SNAKE_CASE_									=						output_from_past[:, 0, random_slice_idx].detach()

      # test that outputs are equal for slice
      assert torch.allclose(_A						,		_A						,		atol=1E-3)


  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						self.prepare_config_and_inputs()
      SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_									=						config_and_inputs

      SCREAMING_SNAKE_CASE_									=						{'input_ids': input_ids, 'attention_mask': attention_mask}
      return config, inputs_dict






@require_torch
class       __snake_case  (    lowerCAmelCase__     , lowerCAmelCase__     , lowerCAmelCase__     , unittest.TestCase       ):
  __lowerCAmelCase     :	Tuple						 =  (TrOCRDecoder, TrOCRForCausalLM) if is_torch_available() else ()
  __lowerCAmelCase     :	Union[str, Any]						 =  (TrOCRForCausalLM,) if is_torch_available() else ()
  __lowerCAmelCase     :	str						 =  {'text-generation': TrOCRForCausalLM} if is_torch_available() else {}
  __lowerCAmelCase     :	Any						 =  True
  __lowerCAmelCase     :	str						 =  False

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						TrOCRStandaloneDecoderModelTester(self						,		is_training=_A)
      SCREAMING_SNAKE_CASE_									=						ConfigTester(self						,		config_class=_A)

  def 						lowerCAmelCase__     (					self):
      pass

  def 						lowerCAmelCase__     (					self):
      pass

  def 						lowerCAmelCase__     (					self):
      pass

  def 						lowerCAmelCase__     (					self):
      self.config_tester.run_common_tests()

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_decoder_model_past(*_A)

  def 						lowerCAmelCase__     (					self):
      return


  @unittest.skip('The model doesn\'t support left padding')  # and it's not used enough to be worth fixing :)
  def 						lowerCAmelCase__     (					self):
      pass


