Update

app.py

@@ -23,28 +23,35 @@ except Exception:  # pylint: disable=broad-except
     logger.warning("Windows, cant run time.tzset()")
 
 model = None
-gc.collect()
+gc.collect()  # for interactive testing
 
 logger.info("start")
 has_cuda = torch.cuda.is_available()
 
+tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False, trust_remote_code=True)
+
 if has_cuda:
     model = AutoModelForCausalLM.from_pretrained(
         "model",  # loc
-        # device_map="auto",
+        device_map="auto",
         torch_dtype=torch.bfloat16,  # pylint: disable=no-member
         load_in_8bit=True,
         trust_remote_code=True,
         # use_ram_optimized_load=False,
         # offload_folder="offload_folder",
-    ).cuda()
+    )  # .cuda()
 else:
-    # model = AutoModel.from_pretrained(model_name, trust_remote_code=True).float()
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name, trust_remote_code=True
-    ).float()
+    try:
+        # model = AutoModel.from_pretrained(model_name, trust_remote_code=True).float()
+        model = AutoModelForCausalLM.from_pretrained(
+            model_name, trust_remote_code=True
+        ).float()
+    except Exception as exc:
+        logger.error(exc)
+        logger.warning("Doesnt seem to load for CPU...")
+        raise SystemExit(1) from exc
 
-model = model.eval()
+    model = model.eval()
 
 rich.print(f"{model=}")