Upload 2 files

app.py

@@ -1,27 +1,25 @@
 import gradio as gr
 import spaces
-from transformers import AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer
 import torch
 import os
 from threading import Thread
 import uuid
 import soundfile as sf
 import numpy as np
-import ctranslate2
-from huggingface_hub import hf_hub_download
+from transformers.generation import TextIteratorStreamer
 
 # Model and Tokenizer Loading
-MODEL_ID = "NexaAIDev/Qwen2-Audio-7B-GGUF"
-TOKENIZER_ID = "NexaAIDev/Qwen2-Audio-7B"  # Use the base model's tokenizer
+MODEL_ID = "Qwen/Qwen-Audio-Chat"
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_ID,
+    torch_dtype=torch.float16,
+    device_map="auto",
+    trust_remote_code=True
+)
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
 
-# Download the GGUF model file
-model_path = hf_hub_download(MODEL_ID, "model.gguf")
-
-# Initialize the model and tokenizer
-generator = ctranslate2.Generator(model_path, device="cuda")
-tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_ID, trust_remote_code=True)
-
-DESCRIPTION = "[Qwen2-Audio-7B Demo](https://huggingface.co/NexaAIDev/Qwen2-Audio-7B-GGUF)"
+DESCRIPTION = "[Qwen-Audio-Chat Demo](https://huggingface.co/Qwen/Qwen-Audio-Chat)"
 
 audio_extensions = (".wav", ".mp3", ".ogg", ".flac")
 
@@ -40,26 +38,55 @@ def qwen_inference(audio_input, text_input=None):
     # Process audio input
     audio_data, sample_rate = process_audio(audio_input)
     
-    # Prepare the prompt
+    # Prepare the messages
     if text_input:
-        prompt = f"Below is an audio clip. {text_input}"
+        query = text_input
     else:
-        prompt = "Please describe what you hear in this audio clip."
+        query = "Please describe what you hear in this audio clip."
 
-    # Tokenize input
-    tokens = tokenizer.encode(prompt)
-    
-    # Generate response
-    results = generator.generate_batch(
-        [tokens],
-        max_length=512,
-        sampling_temperature=0.7,
-        sampling_topk=50,
-        include_prompt_in_result=False
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "audio",
+                    "audio": audio_input,
+                },
+                {
+                    "type": "text",
+                    "text": query,
+                },
+            ],
+        }
+    ]
+
+    # Convert messages to model input format
+    text = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
     )
-    
-    response = tokenizer.decode(results[0].sequences_ids[0])
-    return response
+    model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
+
+    # Set up streamer for real-time output
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+    generation_kwargs = dict(
+        model_inputs,
+        streamer=streamer,
+        max_new_tokens=512,
+        temperature=0.7,
+        do_sample=True
+    )
+
+    # Start generation in a separate thread
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
+
+    # Stream the output
+    buffer = ""
+    for new_text in streamer:
+        buffer += new_text
+        yield buffer
 
 css = """
   #output {

requirements.txt

@@ -4,4 +4,3 @@ transformers>=4.36.0
 soundfile>=0.12.1
 numpy>=1.24.0
 huggingface-hub>=0.19.0
-ctranslate2>=3.23.0