Update app.py

app.py

@@ -3,6 +3,9 @@ from peft import PeftModel
 import transformers
 import gradio as gr
 
+import whisper
+model1 = whisper.load_model("small")
+
 assert (
     "LlamaTokenizer" in transformers._import_structure["models.llama"]
 ), "LLaMA is now in HuggingFace's main branch.\nPlease reinstall it: pip uninstall transformers && pip install git+https://github.com/huggingface/transformers.git"
@@ -84,7 +87,8 @@ if torch.__version__ >= "2":
 
 
 def evaluate(
-    instruction,
+#    instruction,
+    audio,
     input=None,
     temperature=0.1,
     top_p=0.75,
@@ -93,6 +97,24 @@ def evaluate(
     max_new_tokens=128,
     **kwargs,
 ):
+
+    # load audio and pad/trim it to fit 30 seconds
+    audio = whisper.load_audio(audio)
+    audio = whisper.pad_or_trim(audio)
+
+    # make log-Mel spectrogram and move to the same device as the model
+    mel = whisper.log_mel_spectrogram(audio).to(model.device)
+
+    # detect the spoken language
+    _, probs = model.detect_language(mel)
+    print(f"Detected language: {max(probs, key=probs.get)}")
+
+    # decode the audio
+    options = whisper.DecodingOptions()
+    result = whisper.decode(model, mel, options)
+
+    instruction = result.text
+    
     prompt = generate_prompt(instruction, input)
     inputs = tokenizer(prompt, return_tensors="pt")
     input_ids = inputs["input_ids"].to(device)
@@ -119,9 +141,7 @@ def evaluate(
 g = gr.Interface(
     fn=evaluate,
     inputs=[
-        gr.components.Textbox(
-            lines=2, label="Instruction", placeholder="Tell me about alpacas."
-        ),
+        gr.Audio(source="microphone", label = "请上传您喜欢的声音(wav文件)", type="filepath"),
         gr.components.Textbox(lines=2, label="Input", placeholder="none"),
         gr.components.Slider(minimum=0, maximum=1, value=0.1, label="Temperature"),
         gr.components.Slider(minimum=0, maximum=1, value=0.75, label="Top p"),