maybe fp16 works?

utils/inference.py

@@ -45,7 +45,7 @@ def load_tokenizer_and_model(base_model, adapter_model, load_8bit=True):
             )
     else:
         model = LlamaForCausalLM.from_pretrained(
-            base_model, device_map={"": device}, low_cpu_mem_usage=True
+            base_model, device_map={"": device}, low_cpu_mem_usage=True, torch_dtype=torch.float16
         )
         if adapter_model is not None:
             model = PeftModel.from_pretrained(