First model version

main.py

@@ -1,33 +1,19 @@
-import os
-
-from langchain.llms import CTransformers
+from ctransformers import AutoModelForCausalLM
 from fastapi import FastAPI
 from pydantic import BaseModel
 
 file_name = "zephyr-7b-beta.Q4_K_S.gguf"
-config = {
-    "max_new_token": 10,
-    "repetition_penalty": 1.1,
-    "temperature": 0.5,
-    "top_k": 50,
-    "top_p": 0.9,
-    "stream": False,
-    "threads": 3,
-}
-llm = CTransformers(
-    model=file_name,
-    model_type="mistral",
-    lib="avx2",
-    **config
+llm = AutoModelForCausalLM.from_pretrained(file_name,
+    model_type='mistral',
+    max_new_tokens = 1096,
+    threads = 3,
 )
-print(llm)
-
 
+#Pydantic object
 class validation(BaseModel):
     prompt: str
 #Fast API
 
-
 app = FastAPI()
 
 @app.post("/llm_on_cpu")
@@ -36,5 +22,5 @@ async def stream(item: validation):
     E_INST = "</s>"
     user, assistant = "<|user|>", "<|assistant|>"
     prompt = f"{system_prompt}{E_INST}\n{user}\n{item.prompt}{E_INST}\n{assistant}\n"
-    print(prompt)
-    return llm.invoke(prompt)
+
+    return llm(prompt)

requirements.txt

@@ -5,6 +5,4 @@ uvicorn
 requests
 python-dotenv
 ctransformers
-torch
-langchain==0.1.9
-ctransformers
+torch