





'''simple docstring'''







import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin


class 					__a      (lowerCamelCase       ,		unittest.TestCase     ):
							__a					:   int									=       DebertaTokenizer
							__a					:   Optional[int]									=       True
							__a					:   Union[str, Any]									=       DebertaTokenizerFast

							def     UpperCAmelCase__	(							self							:     Union[str, Any]   )					->       Tuple:







													"""simple docstring"""
													super().setUp()

													# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
													UpperCAmelCase_			:      Tuple     						=  [
													    '''l''',
													    '''o''',
													    '''w''',
													    '''e''',
													    '''r''',
													    '''s''',
													    '''t''',
													    '''i''',
													    '''d''',
													    '''n''',
													    '''\u0120''',
													    '''\u0120l''',
													    '''\u0120n''',
													    '''\u0120lo''',
													    '''\u0120low''',
													    '''er''',
													    '''\u0120lowest''',
													    '''\u0120newer''',
													    '''\u0120wider''',
													    '''[UNK]''',
													]
													UpperCAmelCase_			:      Tuple     						=  dict(zip(__magic_name__       ,     range(len(__magic_name__   )   )   )   )
													UpperCAmelCase_			:      Tuple     						=  ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
													UpperCAmelCase_			:      List[str]     						=  {'''unk_token''': '''[UNK]'''}

													UpperCAmelCase_			:      Any     						=  os.path.join(self.tmpdirname       ,     VOCAB_FILES_NAMES['''vocab_file''']   )
													UpperCAmelCase_			:      Tuple     						=  os.path.join(self.tmpdirname       ,     VOCAB_FILES_NAMES['''merges_file''']   )
													with open(self.vocab_file       ,     '''w'''       ,     encoding='''utf-8'''   ) as fp:
																			fp.write(json.dumps(__magic_name__   ) + '''\n'''   )
													with open(self.merges_file       ,     '''w'''       ,     encoding='''utf-8'''   ) as fp:
																			fp.write('''\n'''.join(__magic_name__   )   )

							def     UpperCAmelCase__	(							self							:     Optional[int]       ,     **__magic_name__							:     int   )					->       str:







													"""simple docstring"""
													kwargs.update(self.special_tokens_map   )
													return self.tokenizer_class.from_pretrained(self.tmpdirname       ,     **__magic_name__   )

							def     UpperCAmelCase__	(							self							:     Any       ,     __magic_name__							:     str   )					->       int:







													"""simple docstring"""
													UpperCAmelCase_			:      Dict     						=  '''lower newer'''
													UpperCAmelCase_			:      Optional[Any]     						=  '''lower newer'''
													return input_text, output_text

							def     UpperCAmelCase__	(							self							:     str   )					->       Dict:







													"""simple docstring"""
													UpperCAmelCase_			:      List[Any]     						=  self.get_tokenizer()
													UpperCAmelCase_			:      Tuple     						=  '''lower newer'''
													UpperCAmelCase_			:      List[str]     						=  ['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
													UpperCAmelCase_			:      Tuple     						=  tokenizer.tokenize(__magic_name__   )
													self.assertListEqual(__magic_name__       ,     __magic_name__   )

													UpperCAmelCase_			:      Any     						=  tokens + [tokenizer.unk_token]
													UpperCAmelCase_			:      str     						=  [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
													self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__   )       ,     __magic_name__   )

							def     UpperCAmelCase__	(							self							:     List[str]   )					->       int:







													"""simple docstring"""
													UpperCAmelCase_			:      str     						=  self.get_tokenizer()
													UpperCAmelCase_			:      Optional[int]     						=  tokenizer('''Hello'''       ,     '''World'''   )
													UpperCAmelCase_			:      Any     						=  [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
													self.assertListEqual(tokd['''token_type_ids''']       ,     __magic_name__   )

							@slow
							def     UpperCAmelCase__	(							self							:     Any   )					->       Tuple:







													"""simple docstring"""
													UpperCAmelCase_			:      List[str]     						=  self.tokenizer_class.from_pretrained('''microsoft/deberta-base'''   )

													UpperCAmelCase_			:      Tuple     						=  tokenizer.encode('''sequence builders'''       ,     add_special_tokens=__magic_name__   )
													UpperCAmelCase_			:      Any     						=  tokenizer.encode('''multi-sequence build'''       ,     add_special_tokens=__magic_name__   )

													UpperCAmelCase_			:      Union[str, Any]     						=  tokenizer.encode(
													    '''sequence builders'''       ,     add_special_tokens=__magic_name__       ,     add_prefix_space=__magic_name__   )
													UpperCAmelCase_			:      List[Any]     						=  tokenizer.encode(
													    '''sequence builders'''       ,     '''multi-sequence build'''       ,     add_special_tokens=__magic_name__       ,     add_prefix_space=__magic_name__   )

													UpperCAmelCase_			:      Optional[Any]     						=  tokenizer.build_inputs_with_special_tokens(__magic_name__   )
													UpperCAmelCase_			:      Optional[int]     						=  tokenizer.build_inputs_with_special_tokens(__magic_name__       ,     __magic_name__   )

													assert encoded_sentence == encoded_text_from_decode
													assert encoded_pair == encoded_pair_from_decode



							@slow
							def     UpperCAmelCase__	(							self							:     Any   )					->       Dict:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[int]     						=  [self.tokenizer_class]
													if self.test_rust_tokenizer:
																			tokenizer_classes.append(self.rust_tokenizer_class   )

													for tokenizer_class in tokenizer_classes:
																			UpperCAmelCase_			:      List[str]     						=  tokenizer_class.from_pretrained('''microsoft/deberta-base'''   )

																			UpperCAmelCase_			:      int     						=  [
																			    '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
																			    '''ALBERT incorporates two parameter reduction techniques''',
																			    '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
																			    ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
																			    ''' vocabulary embedding.''',
																			]

																			UpperCAmelCase_			:      Optional[Any]     						=  tokenizer(__magic_name__       ,     padding=__magic_name__   )
																			UpperCAmelCase_			:      List[str]     						=  [tokenizer.decode(__magic_name__       ,     skip_special_tokens=__magic_name__   ) for seq in encoding['''input_ids''']]

																			# fmt: off
																			UpperCAmelCase_			:      Dict     						=  {
																			    '''input_ids''': [
																			        [1, 21_18, 1_11_26, 5_65, 35, 83, 2_51_91, 1_63, 1_88_54, 13, 1_21_56, 12, 1_61_01, 2_53_76, 1_38_07, 9, 2_22_05, 2_78_93, 16_35, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
																			        [1, 21_18, 1_11_26, 5_65, 2_45_36, 80, 4_37_97, 48_78, 73_73, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
																			        [1, 1_33, 78, 65, 16, 10, 37_24, 15_38, 3_31_83, 1_13_03, 4_37_97, 19_38, 4, 8_70, 2_41_65, 2_91_05, 5, 7_39, 3_26_44, 3_31_83, 1_13_03, 3_61_73, 88, 80, 6_50, 78_21, 4_59_40, 6, 52, 25_59, 5, 18_36, 9, 5, 73_97, 1_31_71, 31, 5, 18_36, 9, 3_26_44, 3_31_83, 1_13_03, 4, 2]
																			    ],
																			    '''token_type_ids''': [
																			        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
																			        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
																			        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
																			    ],
																			    '''attention_mask''': [
																			        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
																			        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
																			        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
																			    ]
																			}
																			# fmt: on

																			UpperCAmelCase_			:      str     						=  [
																			    '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
																			    '''ALBERT incorporates two parameter reduction techniques''',
																			    '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
																			    ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
																			    ''' vocabulary embedding.''',
																			]

																			self.assertDictEqual(encoding.data       ,     __magic_name__   )

																			for expected, decoded in zip(__magic_name__       ,     __magic_name__   ):
																									self.assertEqual(__magic_name__       ,     __magic_name__   )












'''simple docstring'''







# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.whisper import WhisperForConditionalGeneration, WhisperProcessor
from .base import PipelineTool


class 					__a      (lowerCamelCase     ):
							__a					:   List[Any]									=       "openai/whisper-base"
							__a					:   Optional[Any]									=       (
							    "This is a tool that transcribes an audio into text. It takes an input named `audio` and returns the "
							    "transcribed text."
							)
							__a					:   Any									=       "transcriber"
							__a					:   str									=       WhisperProcessor
							__a					:   List[Any]									=       WhisperForConditionalGeneration

							__a					:   int									=       ["audio"]
							__a					:   Optional[Any]									=       ["text"]

							def     UpperCAmelCase__	(							self							:     Dict       ,     __magic_name__							:     List[str]   )					->       Optional[int]:







													"""simple docstring"""
													return self.pre_processor(__magic_name__       ,     return_tensors='''pt'''   ).input_features

							def     UpperCAmelCase__	(							self							:     Dict       ,     __magic_name__							:     Dict   )					->       Tuple:







													"""simple docstring"""
													return self.model.generate(inputs=__magic_name__   )



							def     UpperCAmelCase__	(							self							:     List[str]       ,     __magic_name__							:     Dict   )					->       str:







													"""simple docstring"""
													return self.pre_processor.batch_decode(__magic_name__       ,     skip_special_tokens=__magic_name__   )[0]





