Update app.py

app.py

@@ -1,12 +1,25 @@
 import fastapi
+"""
 from fastapi.responses import JSONResponse
 from llama_cpp import Llama
 from time import time
 import logging
 
+"""
+#MODEL_PATH = "./qwen1_5-0_5b-chat-q4_0.gguf" #"./qwen1_5-0_5b-chat-q4_0.gguf"
 
-MODEL_PATH = "./qwen1_5-0_5b-chat-q4_0.gguf" #"./qwen1_5-0_5b-chat-q4_0.gguf"
+import llama_cpp
+import llama_cpp.llama_tokenizer
 
+llama = llama_cpp.Llama.from_pretrained(
+    repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF",
+    filename="*q4_0.gguf",
+    tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained("Qwen/Qwen1.5-0.5B"),
+    verbose=False,
+     n_ctx=4096,
+        n_threads=4,
+        n_gpu_layers=0,
+)
 # Logger setup
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -59,7 +72,7 @@ async def complete(
 ) -> dict:
     try:
         st = time()
-        output = llm.create_chat_completion(
+        output = llama.create_chat_completion(
             messages=[
                 {"role": "system", "content": system},
                 {"role": "user", "content": question},