global

app.py

@@ -300,6 +300,16 @@ def buildmodel(**kwargs):
     for token in added_special_tokens_list:
         quantizer.additional_special_tokens.add(token)
         
+    print(f"before .to(device):{model.device} {model.lm.device}")
+    model = model.to("cuda")
+    print(f"after .to(device):{model.device} {model.lm.device}")
+    model = model.bfloat16()
+    model.eval()
+    # quantizer = quantizer.to("cuda")
+    # tokenizer = tokenizer.to("cuda")
+    # model.lm = model.lm.to("cuda")
+    print(model.lm.device)
+        
     # return model, quantizer, tokenizer
 
 def construction_layout():
@@ -328,15 +338,15 @@ def construction_layout():
     # model.input_embeddings = model.lm.get_input_embeddings()
     # print('after token embeddings to match the tokenizer', 129423)
     
-    print("before .to(device)")
-    model = model.to("cuda")
-    print("after .to(device)")
-    model = model.bfloat16()
-    model.eval()
-    # quantizer = quantizer.to("cuda")
-    # tokenizer = tokenizer.to("cuda")
-    # model.lm = model.lm.to("cuda")
-    print(model.lm.device)
+    # print("before .to(device)")
+    # model = model.to("cuda")
+    # print("after .to(device)")
+    # model = model.bfloat16()
+    # model.eval()
+    # # quantizer = quantizer.to("cuda")
+    # # tokenizer = tokenizer.to("cuda")
+    # # model.lm = model.lm.to("cuda")
+    # print(model.lm.device)
     
     return params_dict["width"], params_dict["height"], device
     # return model, quantizer, tokenizer, params_dict["width"], params_dict["height"], device