q3_k_m

Dockerfile

@@ -32,7 +32,7 @@ ENV HOME=/home/user \
 
 WORKDIR $HOME/app
 
-RUN huggingface-cli download TheBloke/Llama-2-7b-Chat-GGUF llama-2-7b-chat.Q2_K_M.gguf --local-dir . --local-dir-use-symlinks False
+RUN huggingface-cli download TheBloke/Llama-2-7b-Chat-GGUF llama-2-7b-chat.Q3_K_M.gguf --local-dir . --local-dir-use-symlinks False
 
 
 COPY --chown=user . $HOME/app

question_paper.py

@@ -14,7 +14,7 @@ app = FastAPI()
 @app.get("/")
 def llama():
     llm = Llama(
-      model_path="./llama-2-7b-chat.Q2_K_M.gguf",
+      model_path="./llama-2-7b-chat.Q3_K_M.gguf",
       # n_gpu_layers=-1, # Uncomment to use GPU acceleration
       # seed=1337, # Uncomment to set a specific seed
       # n_ctx=2048, # Uncomment to increase the context window