Update main.py

main.py

@@ -3,10 +3,9 @@ from fastapi import FastAPI, Form
 from pydantic import BaseModel
 
 #Model loading
-llm = AutoModelForCausalLM.from_pretrained("zephyr-7b-beta.Q4_K_S.gguf",
-model_type='mistral',
-max_new_tokens = 1096,
-threads = 3,
+llm = AutoModelForCausalLM.from_pretrained("model.gguf",
+model_type='internLM2',
+max_new_tokens = 512
 )
    
 
@@ -20,8 +19,7 @@ app = FastAPI()
 #Zephyr completion
 @app.post("/llm_on_cpu")
 async def stream(item: validation):
-    system_prompt = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.'
     E_INST = "</s>"
     user, assistant = "<|user|>", "<|assistant|>"
-    prompt = f"{system_prompt}{E_INST}\n{user}\n{item.prompt.strip()}{E_INST}\n{assistant}\n"
+    prompt = f"{E_INST}\n{user}\n{item.prompt.strip()}{E_INST}\n{assistant}\n"
     return llm(prompt)