First model version

main.py

@@ -6,13 +6,13 @@ from pydantic import BaseModel
 
 file_name = "zephyr-7b-beta.Q4_K_S.gguf"
 config = {
-    "max_new_token": 1024,
+    "max_new_token": 10,
     "repetition_penalty": 1.1,
     "temperature": 0.5,
     "top_k": 50,
     "top_p": 0.9,
-    "stream": True,
-    "threads": int(os.cpu_count() / 2),
+    "stream": False,
+    "threads": 3,
 }
 llm = CTransformers(
     model=file_name,