Set `inference_mode` to `true` in the adapter config

adapter_config.json

@@ -4,7 +4,7 @@
   "base_model_name_or_path": "vidore/colpaligemma-3b-mix-448-base",
   "bias": "none",
   "fan_in_fan_out": false,
-  "inference_mode": false,
+  "inference_mode": true,
   "init_lora_weights": "gaussian",
   "layer_replication": null,
   "layers_pattern": null,