Update app.py

app.py

@@ -5,6 +5,8 @@ import gradio as gr
 import requests
 import json
 
+from accelerate import Accelerator
+
 SYSTEM_PROMPT = "As a generative chatbot (you are not a GPT but your structure is 50% the same), your primary function is to provide helpful and friendly responses to user queries. Feel free to add some personality, but make sure your responses are accurate and helpful. Your owner and developer is: @Costikoooo (Discord user) other developers are unknown. Your name is Chattybot."
 TITLE = "Chattybot"
 EXAMPLE_INPUT = "hello"
@@ -20,6 +22,12 @@ model = AutoModelForCausalLM.from_pretrained(
 HF_TOKEN = os.getenv("HF_TOKEN")
 HEADERS = {"Authorization": f"Bearer {HF_TOKEN}"}
 
+# Initialize Accelerator
+accelerator = Accelerator()
+
+# Wrap model and tokenizer with accelerator
+model, tokenizer = accelerator.prepare(model, tokenizer)
+
 def build_input_prompt(message, chatbot, system_prompt):
     input_prompt = "\n" + system_prompt + "</s>\n\n"
     for interaction in chatbot:
@@ -32,17 +40,15 @@ def predict_beta(message, chatbot=[], system_prompt=""):
     input_prompt = build_input_prompt(message, chatbot, system_prompt)
     inputs = tokenizer(input_prompt, return_tensors="pt")
 
-    try:
+    with accelerator.device():
         tokens = model.generate(
             inputs["input_ids"],
             max_length=1024,
             temperature=0.8,
             do_sample=True
         )
-        bot_message = tokenizer.decode(tokens[0], skip_special_tokens=True)
-        return bot_message
-    except Exception as e:
-        raise gr.Error(str(e))
+    bot_message = tokenizer.decode(tokens[0], skip_special_tokens=True)
+    return bot_message
 
 def test_preview_chatbot(message, history):
     response = predict_beta(message, history, SYSTEM_PROMPT)