





'''simple docstring'''




import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin






class UpperCAmelCase							(		UpperCAmelCase_					,			unittest.TestCase							):
    _A :			str						   =       TransfoXLTokenizer
    _A :			List[Any]						   =       False
    _A :			Union[str, Any]						   =       False



    def 					__lowerCamelCase						(  self					):
     super().setUp()

     __UpperCAmelCase					=   [
         '<unk>',
         '[CLS]',
         '[SEP]',
         'want',
         'unwanted',
         'wa',
         'un',
         'running',
         ',',
         'low',
         'l',
     ]
     __UpperCAmelCase					=   os.path.join(self.tmpdirname     ,			VOCAB_FILES_NAMES['vocab_file']					)
     with open(self.vocab_file     ,			'w'     ,			encoding='utf-8'					) as vocab_writer:
      vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)



    def 					__lowerCamelCase						(  self     ,			**__A					):
     __UpperCAmelCase					=   True
     return TransfoXLTokenizer.from_pretrained(self.tmpdirname     ,			**__A					)



    def 					__lowerCamelCase						(  self     ,			__A					):
     __UpperCAmelCase					=   '<unk> UNwanted , running'
     __UpperCAmelCase					=   '<unk> unwanted, running'
     return input_text, output_text



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   TransfoXLTokenizer(vocab_file=self.vocab_file     ,			lower_case=__A					)

     __UpperCAmelCase					=   tokenizer.tokenize('<unk> UNwanted , running'					)
     self.assertListEqual(__A     ,			['<unk>', 'unwanted', ',', 'running']					)

     self.assertListEqual(tokenizer.convert_tokens_to_ids(__A					)     ,			[0, 4, 8, 7]					)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   TransfoXLTokenizer(lower_case=__A					)

     self.assertListEqual(
         tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '					)     ,			['hello', '!', 'how', 'are', 'you', '?']					)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   TransfoXLTokenizer(lower_case=__A					)

     self.assertListEqual(
         tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '					)     ,			['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   TransfoXLTokenizer(lower_case=__A					)
     __UpperCAmelCase					=   'Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'
     __UpperCAmelCase					=   [
         'Hello',
         '(',
         'bracket',
         ')',
         'and',
         'side',
         '@-@',
         'scrolled',
         '[',
         'and',
         ']',
         'Henry',
         '\'s',
         '$',
         '5',
         '@,@',
         '000',
         'with',
         '3',
         '@.@',
         '34',
         'm',
         '.',
         'What',
         '\'s',
         'up',
         '!',
         '?',
     ]

     self.assertListEqual(tokenizer.tokenize(__A					)     ,			__A					)

     self.assertEqual(tokenizer.convert_tokens_to_string(__A					)     ,			__A					)




    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   self.get_tokenizer()
     __UpperCAmelCase					=   len(__A					)

     tokenizer.add_tokens(['new1', 'new2']					)
     tokenizer.move_added_token('new1'     ,			1					)

     # Check that moved token is not copied (duplicate)
     self.assertEqual(len(__A					)     ,			original_len + 2					)
     # Check that token is moved to specified id
     self.assertEqual(tokenizer.encode('new1'					)     ,			[1]					)
     self.assertEqual(tokenizer.decode([1]					)     ,			'new1'					)












'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_A:					str							=				{
    """configuration_whisper""": ["""WHISPER_PRETRAINED_CONFIG_ARCHIVE_MAP""", """WhisperConfig""", """WhisperOnnxConfig"""],
    """feature_extraction_whisper""": ["""WhisperFeatureExtractor"""],
    """processing_whisper""": ["""WhisperProcessor"""],
    """tokenization_whisper""": ["""WhisperTokenizer"""],
}

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _A:					Union[str, Any]							=				["""WhisperTokenizerFast"""]

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _A:					Optional[Any]							=				[
        """WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """WhisperForConditionalGeneration""",
        """WhisperModel""",
        """WhisperPreTrainedModel""",
        """WhisperForAudioClassification""",
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _A:					Dict							=				[
        """TF_WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """TFWhisperForConditionalGeneration""",
        """TFWhisperModel""",
        """TFWhisperPreTrainedModel""",
    ]

try:
    if not is_flax_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _A:					Union[str, Any]							=				[
        """FlaxWhisperForConditionalGeneration""",
        """FlaxWhisperModel""",
        """FlaxWhisperPreTrainedModel""",
        """FlaxWhisperForAudioClassification""",
    ]


if TYPE_CHECKING:
    from .configuration_whisper import WHISPER_PRETRAINED_CONFIG_ARCHIVE_MAP, WhisperConfig, WhisperOnnxConfig
    from .feature_extraction_whisper import WhisperFeatureExtractor
    from .processing_whisper import WhisperProcessor
    from .tokenization_whisper import WhisperTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_whisper_fast import WhisperTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_whisper import (
            WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST,
            WhisperForAudioClassification,
            WhisperForConditionalGeneration,
            WhisperModel,
            WhisperPreTrainedModel,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_whisper import (
            TF_WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFWhisperForConditionalGeneration,
            TFWhisperModel,
            TFWhisperPreTrainedModel,
        )

    try:
        if not is_flax_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_flax_whisper import (
            FlaxWhisperForAudioClassification,
            FlaxWhisperForConditionalGeneration,
            FlaxWhisperModel,
            FlaxWhisperPreTrainedModel,
        )

else:
    import sys

    _A:					int							=				_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





