test

main.py

@@ -2,12 +2,11 @@ from ctransformers import AutoModelForCausalLM
 from fastapi import FastAPI
 from pydantic import BaseModel
 
-file_name = "gemma-2b.Q2_K.gguf"
+file_name = "zephyr-7b-beta.Q4_K_S.gguf"
 llm = AutoModelForCausalLM.from_pretrained(file_name,
     model_type='mistral',
-    max_new_tokens=2_096,
-    context_length=8_000,
-    threads=3,
+    max_new_tokens=2096,
+    threads=8000,
 )
 
 #Pydantic object