Using Q8_0 instead.

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 models/final-gemma2b_SA-Q5_K.gguf filter=lfs diff=lfs merge=lfs -text
+models/final-gemma2b_SA-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text

app/main.py

@@ -8,7 +8,7 @@ from llama_cpp import Llama
 
 print("Loading model...")
 llm = Llama(
-      model_path="/models/final-gemma2b_SA-Q5_K.gguf",
+      model_path="/models/final-gemma2b_SA-Q8_0.gguf",
       # n_gpu_layers=28, # Uncomment to use GPU acceleration
       # seed=1337, # Uncomment to set a specific seed
       # n_ctx=2048, # Uncomment to increase the context window

models/final-gemma2b_SA-Q8_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8770ad0c0a61bdda3c6e2c07b2db9381300b7e9699affa1419bd4ebd8256edd5
+size 2669351840