

import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__  :      Optional[Any]			=    get_tests_dir("fixtures/test_sentencepiece_bpe_char.model")



@require_sentencepiece
@require_tokenizers
class       __snake_case  (    lowerCAmelCase__     , unittest.TestCase       ):
  __lowerCAmelCase     :	Tuple						 =  SpeechTaTokenizer
  __lowerCAmelCase     :	int						 =  False
  __lowerCAmelCase     :	int						 =  True

  def 						lowerCAmelCase__     (					self):
      super().setUp()

      # We have a SentencePiece fixture for testing
      SCREAMING_SNAKE_CASE_									=						SpeechTaTokenizer(_A)

      SCREAMING_SNAKE_CASE_									=						AddedToken('<mask>'						,		lstrip=_A						,		rstrip=_A)
      SCREAMING_SNAKE_CASE_									=						mask_token
      tokenizer.add_special_tokens({'mask_token': mask_token})
      tokenizer.add_tokens(['<ctc_blank>'])

      tokenizer.save_pretrained(self.tmpdirname)

  def 						lowerCAmelCase__     (					self						,		_A):
      SCREAMING_SNAKE_CASE_									=						'this is a test'
      SCREAMING_SNAKE_CASE_									=						'this is a test'
      return input_text, output_text

  def 						lowerCAmelCase__     (					self						,		_A						,		_A=False						,		_A=20						,		_A=5):
      SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_									=						self.get_input_output_texts(_A)
      SCREAMING_SNAKE_CASE_									=						tokenizer.encode(_A						,		add_special_tokens=_A)
      SCREAMING_SNAKE_CASE_									=						tokenizer.decode(_A						,		clean_up_tokenization_spaces=_A)
      return text, ids

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						'<pad>'
      SCREAMING_SNAKE_CASE_									=						1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A)						,		_A)
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A)						,		_A)

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						list(self.get_tokenizer().get_vocab().keys())

      self.assertEqual(vocab_keys[0]						,		'<s>')
      self.assertEqual(vocab_keys[1]						,		'<pad>')
      self.assertEqual(vocab_keys[-4]						,		'œ')
      self.assertEqual(vocab_keys[-2]						,		'<mask>')
      self.assertEqual(vocab_keys[-1]						,		'<ctc_blank>')
      self.assertEqual(len(_A)						,		81)

  def 						lowerCAmelCase__     (					self):
      self.assertEqual(self.get_tokenizer().vocab_size						,		79)

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						self.get_tokenizers(do_lower_case=_A)
      for tokenizer in tokenizers:
          with self.subTest(f"""{tokenizer.__class__.__name__}"""):
              SCREAMING_SNAKE_CASE_									=						tokenizer.vocab_size
              SCREAMING_SNAKE_CASE_									=						len(_A)

              self.assertNotEqual(_A						,		0)

              # We usually have added tokens from the start in tests because our vocab fixtures are
              # smaller than the original vocabs - let's not assert this
              # self.assertEqual(vocab_size, all_size)

              SCREAMING_SNAKE_CASE_									=						['aaaaa bbbbbb', 'cccccccccdddddddd']
              SCREAMING_SNAKE_CASE_									=						tokenizer.add_tokens(_A)
              SCREAMING_SNAKE_CASE_									=						tokenizer.vocab_size
              SCREAMING_SNAKE_CASE_									=						len(_A)

              self.assertNotEqual(_A						,		0)
              self.assertEqual(_A						,		_A)
              self.assertEqual(_A						,		len(_A))
              self.assertEqual(_A						,		all_size + len(_A))

              SCREAMING_SNAKE_CASE_									=						tokenizer.encode('aaaaa bbbbbb low cccccccccdddddddd l'						,		add_special_tokens=_A)

              self.assertGreaterEqual(len(_A)						,		4)
              self.assertGreater(tokens[0]						,		tokenizer.vocab_size - 1)
              self.assertGreater(tokens[-3]						,		tokenizer.vocab_size - 1)

              SCREAMING_SNAKE_CASE_									=						{'eos_token': '>>>>|||<||<<|<<', 'pad_token': '<<<<<|||>|>>>>|>'}
              SCREAMING_SNAKE_CASE_									=						tokenizer.add_special_tokens(_A)
              SCREAMING_SNAKE_CASE_									=						tokenizer.vocab_size
              SCREAMING_SNAKE_CASE_									=						len(_A)

              self.assertNotEqual(_A						,		0)
              self.assertEqual(_A						,		_A)
              self.assertEqual(_A						,		len(_A))
              self.assertEqual(_A						,		all_size_a + len(_A))

              SCREAMING_SNAKE_CASE_									=						tokenizer.encode(
                  '>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'						,		add_special_tokens=_A)

              self.assertGreaterEqual(len(_A)						,		6)
              self.assertGreater(tokens[0]						,		tokenizer.vocab_size - 1)
              self.assertGreater(tokens[0]						,		tokens[1])
              self.assertGreater(tokens[-3]						,		tokenizer.vocab_size - 1)
              self.assertGreater(tokens[-3]						,		tokens[-4])
              self.assertEqual(tokens[0]						,		tokenizer.eos_token_id)
              self.assertEqual(tokens[-3]						,		tokenizer.pad_token_id)

  def 						lowerCAmelCase__     (					self):
      pass

  def 						lowerCAmelCase__     (					self):
      pass

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						self.get_tokenizer()

      SCREAMING_SNAKE_CASE_									=						tokenizer.tokenize('This is a test')
      # fmt: off
      self.assertListEqual(_A						,		[SPIECE_UNDERLINE, 'T', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'a', SPIECE_UNDERLINE, 't', 'e', 's', 't'])
      # fmt: on

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(_A)						,		[4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]						,		)

      SCREAMING_SNAKE_CASE_									=						tokenizer.tokenize('I was born in 92000, and this is falsé.')
      self.assertListEqual(
          _A						,		[SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '92000', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.'])

      SCREAMING_SNAKE_CASE_									=						tokenizer.convert_tokens_to_ids(_A)
      # fmt: off
      self.assertListEqual(_A						,		[4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26])
      # fmt: on

      SCREAMING_SNAKE_CASE_									=						tokenizer.convert_ids_to_tokens(_A)
      self.assertListEqual(
          _A						,		[SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '<unk>', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.'])


  @slow
  def 						lowerCAmelCase__     (					self):
      # Use custom sequence because this tokenizer does not handle numbers.
      SCREAMING_SNAKE_CASE_									=						[
          'Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '
          'general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '
          'Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '
          'models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.',
          'BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '
          'conditioning on both left and right context in all layers.',
          'The quick brown fox jumps over the lazy dog.',
      ]

      # fmt: off
      SCREAMING_SNAKE_CASE_									=						{
          'input_ids': [
              [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
              [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
              [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
          ],
          'attention_mask': [
              [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
              [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
              [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
          ]
      }
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=_A						,		model_name='microsoft/speecht5_asr'						,		revision='c5ef64c71905caeccde0e4462ef3f9077224c524'						,		sequences=_A						,		)





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__  :      Union[str, Any]			=    logging.get_logger(__name__)


UpperCamelCase__  :      Optional[Any]			=    {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

# See all BART models at https://huggingface.co/models?filter=bart
UpperCamelCase__  :      List[str]			=    {
    "vocab_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/vocab.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/vocab.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json",
    },
    "merges_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/merges.txt",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/merges.txt",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt",
    },
}

UpperCamelCase__  :      str			=    {
    "facebook/bart-base": 1_024,
    "facebook/bart-large": 1_024,
    "facebook/bart-large-mnli": 1_024,
    "facebook/bart-large-cnn": 1_024,
    "facebook/bart-large-xsum": 1_024,
    "yjernite/bart_eli5": 1_024,
}


@lru_cache()
def 					_UpperCAmelCase       (  ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						(
        list(range(ord('!'       ) ,   ord('~'       ) + 1       )       ) + list(range(ord('¡'       ) ,   ord('¬'       ) + 1       )       ) + list(range(ord('®'       ) ,   ord('ÿ'       ) + 1       )       )
    )
    SCREAMING_SNAKE_CASE_									=						bs[:]
    SCREAMING_SNAKE_CASE_									=						0
    for b in range(2**8       ):
        if b not in bs:
            bs.append(_SCREAMING_SNAKE_CASE       )
            cs.append(2**8 + n       )
            n += 1
    SCREAMING_SNAKE_CASE_									=						[chr(_SCREAMING_SNAKE_CASE       ) for n in cs]
    return dict(zip(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )       )


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		List[str]       ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						set()
    SCREAMING_SNAKE_CASE_									=						word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)       )
        SCREAMING_SNAKE_CASE_									=						char
    return pairs





class       __snake_case  (    lowerCAmelCase__       ):
  __lowerCAmelCase     :	str						 =  VOCAB_FILES_NAMES
  __lowerCAmelCase     :	Any						 =  PRETRAINED_VOCAB_FILES_MAP
  __lowerCAmelCase     :	Dict						 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowerCAmelCase     :	List[Any]						 =  ['input_ids', 'attention_mask']

  def __init__(					self						,		_A						,		_A						,		_A="replace"						,		_A="<s>"						,		_A="</s>"						,		_A="</s>"						,		_A="<s>"						,		_A="<unk>"						,		_A="<pad>"						,		_A="<mask>"						,		_A=False						,		**_A						,		):
      SCREAMING_SNAKE_CASE_									=						AddedToken(_A						,		lstrip=_A						,		rstrip=_A) if isinstance(_A						,		_A) else bos_token
      SCREAMING_SNAKE_CASE_									=						AddedToken(_A						,		lstrip=_A						,		rstrip=_A) if isinstance(_A						,		_A) else eos_token
      SCREAMING_SNAKE_CASE_									=						AddedToken(_A						,		lstrip=_A						,		rstrip=_A) if isinstance(_A						,		_A) else sep_token
      SCREAMING_SNAKE_CASE_									=						AddedToken(_A						,		lstrip=_A						,		rstrip=_A) if isinstance(_A						,		_A) else cls_token
      SCREAMING_SNAKE_CASE_									=						AddedToken(_A						,		lstrip=_A						,		rstrip=_A) if isinstance(_A						,		_A) else unk_token
      SCREAMING_SNAKE_CASE_									=						AddedToken(_A						,		lstrip=_A						,		rstrip=_A) if isinstance(_A						,		_A) else pad_token

      # Mask token behave like a normal word, i.e. include the space before it
      SCREAMING_SNAKE_CASE_									=						AddedToken(_A						,		lstrip=_A						,		rstrip=_A) if isinstance(_A						,		_A) else mask_token

      super().__init__(
          errors=_A						,		bos_token=_A						,		eos_token=_A						,		unk_token=_A						,		sep_token=_A						,		cls_token=_A						,		pad_token=_A						,		mask_token=_A						,		add_prefix_space=_A						,		**_A						,		)

      with open(_A						,		encoding='utf-8') as vocab_handle:
          SCREAMING_SNAKE_CASE_									=						json.load(_A)
      SCREAMING_SNAKE_CASE_									=						{v: k for k, v in self.encoder.items()}
      SCREAMING_SNAKE_CASE_									=						errors  # how to handle errors in decoding
      SCREAMING_SNAKE_CASE_									=						bytes_to_unicode()
      SCREAMING_SNAKE_CASE_									=						{v: k for k, v in self.byte_encoder.items()}
      with open(_A						,		encoding='utf-8') as merges_handle:
          SCREAMING_SNAKE_CASE_									=						merges_handle.read().split('\n')[1:-1]
      SCREAMING_SNAKE_CASE_									=						[tuple(merge.split()) for merge in bpe_merges]
      SCREAMING_SNAKE_CASE_									=						dict(zip(_A						,		range(len(_A))))
      SCREAMING_SNAKE_CASE_									=						{}
      SCREAMING_SNAKE_CASE_									=						add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      SCREAMING_SNAKE_CASE_									=						re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+')

  @property
  def 						lowerCAmelCase__     (					self):
      return len(self.encoder)

  def 						lowerCAmelCase__     (					self):
      return dict(self.encoder						,		**self.added_tokens_encoder)

  def 						lowerCAmelCase__     (					self						,		_A):
      if token in self.cache:
          return self.cache[token]
      SCREAMING_SNAKE_CASE_									=						tuple(_A)
      SCREAMING_SNAKE_CASE_									=						get_pairs(_A)

      if not pairs:
          return token

      while True:
          SCREAMING_SNAKE_CASE_									=						min(_A						,		key=lambda _A: self.bpe_ranks.get(_A						,		float('inf')))
          if bigram not in self.bpe_ranks:
              break
          SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_									=						bigram
          SCREAMING_SNAKE_CASE_									=						[]
          SCREAMING_SNAKE_CASE_									=						0
          while i < len(_A):
              try:
                  SCREAMING_SNAKE_CASE_									=						word.index(_A						,		_A)
              except ValueError:
                  new_word.extend(word[i:])
                  break
              else:
                  new_word.extend(word[i:j])
                  SCREAMING_SNAKE_CASE_									=						j

              if word[i] == first and i < len(_A) - 1 and word[i + 1] == second:
                  new_word.append(first + second)
                  i += 2
              else:
                  new_word.append(word[i])
                  i += 1
          SCREAMING_SNAKE_CASE_									=						tuple(_A)
          SCREAMING_SNAKE_CASE_									=						new_word
          if len(_A) == 1:
              break
          else:
              SCREAMING_SNAKE_CASE_									=						get_pairs(_A)
      SCREAMING_SNAKE_CASE_									=						' '.join(_A)
      SCREAMING_SNAKE_CASE_									=						word
      return word

  def 						lowerCAmelCase__     (					self						,		_A):
      SCREAMING_SNAKE_CASE_									=						[]
      for token in re.findall(self.pat						,		_A):
          SCREAMING_SNAKE_CASE_									=						''.join(
              self.byte_encoder[b] for b in token.encode('utf-8'))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
          bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_A).split(' '))
      return bpe_tokens

  def 						lowerCAmelCase__     (					self						,		_A):
      return self.encoder.get(_A						,		self.encoder.get(self.unk_token))

  def 						lowerCAmelCase__     (					self						,		_A):
      return self.decoder.get(_A)

  def 						lowerCAmelCase__     (					self						,		_A):
      SCREAMING_SNAKE_CASE_									=						''.join(_A)
      SCREAMING_SNAKE_CASE_									=						bytearray([self.byte_decoder[c] for c in text]).decode('utf-8'						,		errors=self.errors)
      return text

  def 						lowerCAmelCase__     (					self						,		_A						,		_A = None):
      if not os.path.isdir(_A):
          logger.error(f"""Vocabulary path ({save_directory}) should be a directory""")
          return
      SCREAMING_SNAKE_CASE_									=						os.path.join(
          _A						,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
      SCREAMING_SNAKE_CASE_									=						os.path.join(
          _A						,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

      with open(_A						,		'w'						,		encoding='utf-8') as f:
          f.write(json.dumps(self.encoder						,		indent=2						,		sort_keys=_A						,		ensure_ascii=_A) + '\n')

      SCREAMING_SNAKE_CASE_									=						0
      with open(_A						,		'w'						,		encoding='utf-8') as writer:
          writer.write('#version: 0.2\n')
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,		key=lambda _A: kv[1]):
              if index != token_index:
                  logger.warning(
                      f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                      ' Please check that the tokenizer is not corrupted!')
                  SCREAMING_SNAKE_CASE_									=						token_index
              writer.write(' '.join(_A) + '\n')
              index += 1

      return vocab_file, merge_file

  def 						lowerCAmelCase__     (					self						,		_A						,		_A = None):
      if token_ids_a is None:
          return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      SCREAMING_SNAKE_CASE_									=						[self.cls_token_id]
      SCREAMING_SNAKE_CASE_									=						[self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep

  def 						lowerCAmelCase__     (					self						,		_A						,		_A = None						,		_A = False):
      if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=_A						,		token_ids_a=_A						,		already_has_special_tokens=_A)

      if token_ids_a is None:
          return [1] + ([0] * len(_A)) + [1]
      return [1] + ([0] * len(_A)) + [1, 1] + ([0] * len(_A)) + [1]

  def 						lowerCAmelCase__     (					self						,		_A						,		_A = None):
      SCREAMING_SNAKE_CASE_									=						[self.sep_token_id]
      SCREAMING_SNAKE_CASE_									=						[self.cls_token_id]

      if token_ids_a is None:
          return len(cls + token_ids_a + sep) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]


  def 						lowerCAmelCase__     (					self						,		_A						,		_A=False						,		**_A):
      SCREAMING_SNAKE_CASE_									=						kwargs.pop('add_prefix_space'						,		self.add_prefix_space)
      if (is_split_into_words or add_prefix_space) and (len(_A) > 0 and not text[0].isspace()):
          SCREAMING_SNAKE_CASE_									=						' ' + text
      return (text, kwargs)


