Load in 4bits

app.py

@@ -71,7 +71,8 @@ def get_interface(model_name: str):
             additional_model_config={
                 "device_map": "auto" if has_cuda else "cpu",
                 "quantization_config": BitsAndBytesConfig(
-                    load_in_8bit=True
+                    load_in_4bit=True,
+                    llm_int8_enable_fp32_cpu_offload=True
                 ) if has_cuda else None,
             },
             **model_config