Update app.py

app.py

@@ -1,13 +1,13 @@
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoTokenizer
 from vllm import LLM, SamplingParams
 
-# Load the model and tokenizer from Hugging Face 
+# Load the model and tokenizer from Hugging Face
 model_name = "facebook/opt-125m"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 
 # Initialize vLLM with CPU-only configuration
-vllm_model = LLM(model=model_name, tensor_parallel_size=1, device="cpu")
+vllm_model = LLM(model=model_name, tensor_parallel_size=1, device="cpu", async_output=False)
 
 def generate_response(prompt, max_tokens, temperature, top_p):
     # Tokenize the prompt
@@ -20,8 +20,11 @@ def generate_response(prompt, max_tokens, temperature, top_p):
         top_p=top_p,
     )
     
-    # Generate text using vLLM
-    output = vllm_model.generate(inputs["input_ids"], sampling_params)
+    # Generate text using vLLM (synchronous mode)
+    try:
+        output = vllm_model.generate(inputs["input_ids"], sampling_params)
+    except NotImplementedError as e:
+        return f"Error: {e}. Ensure that async_output is supported or disabled."
     
     # Decode the generated tokens to text
     generated_text = tokenizer.decode(output[0]["token_ids"], skip_special_tokens=True)