"""simple docstring"""


import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_:					Union[str, Any]     =					get_tests_dir("fixtures/test_sentencepiece_bpe_char.model")






@require_sentencepiece
@require_tokenizers
class    a__    (   __lowerCamelCase					,					unittest.TestCase	):
   snake_case_							=      SpeechTaTokenizer
   snake_case_							=      False
   snake_case_							=      True


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        super().setUp()

        # We have a SentencePiece fixture for testing
        lowercase__				 =							SpeechTaTokenizer(UpperCamelCase_  )

        lowercase__				 =							AddedToken("<mask>",				lstrip=UpperCamelCase_,				rstrip=UpperCamelCase_  )
        lowercase__				 =							mask_token
        tokenizer.add_special_tokens({"mask_token": mask_token}  )
        tokenizer.add_tokens(["<ctc_blank>"]  )

        tokenizer.save_pretrained(self.tmpdirname  )


   def 	snake_case__	(     self,				_UpperCAmelCase  ):
        '''simple docstring'''



        lowercase__				 =							"this is a test"
        lowercase__				 =							"this is a test"
        return input_text, output_text


   def 	snake_case__	(     self,				_UpperCAmelCase,				_UpperCAmelCase=False,				_UpperCAmelCase=20,				_UpperCAmelCase=5  ):
        '''simple docstring'''



        lowercase__ ,						lowercase__				 =							self.get_input_output_texts(UpperCamelCase_  )
        lowercase__				 =							tokenizer.encode(UpperCamelCase_,				add_special_tokens=UpperCamelCase_  )
        lowercase__				 =							tokenizer.decode(UpperCamelCase_,				clean_up_tokenization_spaces=UpperCamelCase_  )
        return text, ids


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							"<pad>"
        lowercase__				 =							1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase_  ),				UpperCamelCase_  )
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase_  ),				UpperCamelCase_  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							list(self.get_tokenizer().get_vocab().keys()  )

        self.assertEqual(vocab_keys[0],				"<s>"  )
        self.assertEqual(vocab_keys[1],				"<pad>"  )
        self.assertEqual(vocab_keys[-4],				"œ"  )
        self.assertEqual(vocab_keys[-2],				"<mask>"  )
        self.assertEqual(vocab_keys[-1],				"<ctc_blank>"  )
        self.assertEqual(len(UpperCamelCase_  ),				81  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        self.assertEqual(self.get_tokenizer().vocab_size,				79  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							self.get_tokenizers(do_lower_case=UpperCamelCase_  )
        for tokenizer in tokenizers:
             with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
                  lowercase__				 =							tokenizer.vocab_size
                  lowercase__				 =							len(UpperCamelCase_  )

                  self.assertNotEqual(UpperCamelCase_,				0  )

                  # We usually have added tokens from the start in tests because our vocab fixtures are
                  # smaller than the original vocabs - let's not assert this
                  # self.assertEqual(vocab_size, all_size)

                  lowercase__				 =							["aaaaa bbbbbb", "cccccccccdddddddd"]
                  lowercase__				 =							tokenizer.add_tokens(UpperCamelCase_  )
                  lowercase__				 =							tokenizer.vocab_size
                  lowercase__				 =							len(UpperCamelCase_  )

                  self.assertNotEqual(UpperCamelCase_,				0  )
                  self.assertEqual(UpperCamelCase_,				UpperCamelCase_  )
                  self.assertEqual(UpperCamelCase_,				len(UpperCamelCase_  )  )
                  self.assertEqual(UpperCamelCase_,				all_size + len(UpperCamelCase_  )  )

                  lowercase__				 =							tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l",				add_special_tokens=UpperCamelCase_  )

                  self.assertGreaterEqual(len(UpperCamelCase_  ),				4  )
                  self.assertGreater(tokens[0],				tokenizer.vocab_size - 1  )
                  self.assertGreater(tokens[-3],				tokenizer.vocab_size - 1  )

                  lowercase__				 =							{"eos_token": ">>>>|||<||<<|<<", "pad_token": "<<<<<|||>|>>>>|>"}
                  lowercase__				 =							tokenizer.add_special_tokens(UpperCamelCase_  )
                  lowercase__				 =							tokenizer.vocab_size
                  lowercase__				 =							len(UpperCamelCase_  )

                  self.assertNotEqual(UpperCamelCase_,				0  )
                  self.assertEqual(UpperCamelCase_,				UpperCamelCase_  )
                  self.assertEqual(UpperCamelCase_,				len(UpperCamelCase_  )  )
                  self.assertEqual(UpperCamelCase_,				all_size_a + len(UpperCamelCase_  )  )

                  lowercase__				 =							tokenizer.encode(
                      ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l",				add_special_tokens=UpperCamelCase_  )

                  self.assertGreaterEqual(len(UpperCamelCase_  ),				6  )
                  self.assertGreater(tokens[0],				tokenizer.vocab_size - 1  )
                  self.assertGreater(tokens[0],				tokens[1]  )
                  self.assertGreater(tokens[-3],				tokenizer.vocab_size - 1  )
                  self.assertGreater(tokens[-3],				tokens[-4]  )
                  self.assertEqual(tokens[0],				tokenizer.eos_token_id  )
                  self.assertEqual(tokens[-3],				tokenizer.pad_token_id  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        pass


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        pass


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							self.get_tokenizer()

        lowercase__				 =							tokenizer.tokenize("This is a test"  )
        # fmt: off
        self.assertListEqual(UpperCamelCase_,				[SPIECE_UNDERLINE, "T", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "a", SPIECE_UNDERLINE, "t", "e", "s", "t"]  )
        # fmt: on

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(UpperCamelCase_  ),				[4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6],				)

        lowercase__				 =							tokenizer.tokenize("I was born in 92000, and this is falsé."  )
        self.assertListEqual(
            UpperCamelCase_,				[SPIECE_UNDERLINE, "I", SPIECE_UNDERLINE, "w", "a", "s", SPIECE_UNDERLINE, "b", "o", "r", "n", SPIECE_UNDERLINE, "i", "n", SPIECE_UNDERLINE, "92000", ",", SPIECE_UNDERLINE, "a", "n", "d", SPIECE_UNDERLINE, "t", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "f", "a", "l", "s", "é", "."]  )

        lowercase__				 =							tokenizer.convert_tokens_to_ids(UpperCamelCase_  )
        # fmt: off
        self.assertListEqual(UpperCamelCase_,				[4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]  )
        # fmt: on

        lowercase__				 =							tokenizer.convert_ids_to_tokens(UpperCamelCase_  )
        self.assertListEqual(
            UpperCamelCase_,				[SPIECE_UNDERLINE, "I", SPIECE_UNDERLINE, "w", "a", "s", SPIECE_UNDERLINE, "b", "o", "r", "n", SPIECE_UNDERLINE, "i", "n", SPIECE_UNDERLINE, "<unk>", ",", SPIECE_UNDERLINE, "a", "n", "d", SPIECE_UNDERLINE, "t", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "f", "a", "l", "s", "é", "."]  )


   @slow
   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							[
            "Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides "
            "general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural "
            "Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained "
            "models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.",
            "BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly "
            "conditioning on both left and right context in all layers.",
            "The quick brown fox jumps over the lazy dog.",
        ]

        # fmt: off
        lowercase__				 =							{
            "input_ids": [
                [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
                [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
            ],
            "attention_mask": [
                [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
            ]
        }
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=UpperCamelCase_,				model_name="microsoft/speecht5_asr",				revision="c5ef64c71905caeccde0e4462ef3f9077224c524",				sequences=UpperCamelCase_,				)





"""simple docstring"""


import os
import sys

lowerCAmelCase_:					Any     =					os.path.join(os.path.dirname(__file__), "src")
sys.path.append(SRC_DIR)


from transformers import (
    AutoConfig,
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForMaskedLM,
    AutoModelForQuestionAnswering,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    add_start_docstrings,
)


lowerCAmelCase_:					Union[str, Any]     =					[
    "torch",
    "numpy",
    "tokenizers",
    "filelock",
    "requests",
    "tqdm",
    "regex",
    "sentencepiece",
    "sacremoses",
    "importlib_metadata",
    "huggingface_hub",
]







@add_start_docstrings(AutoConfig.__doc__      )
def 					__a   (		*A    ,				**A      ):






     '''simple docstring'''



     return AutoConfig.from_pretrained(*A    ,				**A      )







@add_start_docstrings(AutoTokenizer.__doc__      )
def 					__a   (		*A    ,				**A      ):






     '''simple docstring'''



     return AutoTokenizer.from_pretrained(*A    ,				**A      )







@add_start_docstrings(AutoModel.__doc__      )
def 					__a   (		*A    ,				**A      ):






     '''simple docstring'''



     return AutoModel.from_pretrained(*A    ,				**A      )







@add_start_docstrings(AutoModelForCausalLM.__doc__      )
def 					__a   (		*A    ,				**A      ):






     '''simple docstring'''



     return AutoModelForCausalLM.from_pretrained(*A    ,				**A      )







@add_start_docstrings(AutoModelForMaskedLM.__doc__      )
def 					__a   (		*A    ,				**A      ):






     '''simple docstring'''



     return AutoModelForMaskedLM.from_pretrained(*A    ,				**A      )







@add_start_docstrings(AutoModelForSequenceClassification.__doc__      )
def 					__a   (		*A    ,				**A      ):






     '''simple docstring'''



     return AutoModelForSequenceClassification.from_pretrained(*A    ,				**A      )







@add_start_docstrings(AutoModelForQuestionAnswering.__doc__      )
def 					__a   (		*A    ,				**A      ):






     '''simple docstring'''



     return AutoModelForQuestionAnswering.from_pretrained(*A    ,				**A      )




