Upload folder using huggingface_hub

demo.py

@@ -40,8 +40,7 @@ def load_model():
     logger.info("Loading model...")
     model = AutoModelForCausalLM.from_pretrained(
         "Zwounds/boolean-search-model",
-        torch_dtype=torch.float32,
-        load_in_4bit=False
+        load_in_8bit=True
     )
     tokenizer = AutoTokenizer.from_pretrained("Zwounds/boolean-search-model")
     tokenizer.use_default_system_prompt = False