debug-seed

VADER-VideoCrafter/lvdm/models/samplers/ddim.py

@@ -152,8 +152,10 @@ class DDIMSampler(object):
             img = torch.randn(shape, device=device)
         else:
             img = x_T
-
-        print('random seed debug: ', torch.randn(100).sum())
+        
+        print("x_T: ", x_T)
+        print("shape: ", shape)
+        print('random seed debug: ', torch.randn(100, device=device).sum())
         print("Debug initial noise: ", img.sum().item())
         print("noise device: ", img.device)
         

VADER-VideoCrafter/scripts/main/train_t2v_lora.py

@@ -589,7 +589,7 @@ def run_training(args, model, **kwargs):
         # load the pretrained LoRA model
         peft.set_peft_model_state_dict(peft_model, torch.load(args.lora_ckpt_path))
 
-    print('random seed debug: ', torch.randn(100).sum())
+    print('random seed debug: ', torch.randn(100, device=accelerator.device).sum())
     print("precision: ", peft_model.dtype)
     # precision of first_stage_model
     print("precision of first_stage_model: ", peft_model.first_stage_model.dtype)
@@ -639,7 +639,6 @@ def run_training(args, model, **kwargs):
             if isinstance(prompts, str):
                 prompts = [prompts]
             
-            seed_everything_self(args.seed)
             with accelerator.autocast():            # mixed precision
                 if isinstance(peft_model, torch.nn.parallel.DistributedDataParallel):
                     text_emb = peft_model.module.get_learned_conditioning(prompts).to(accelerator.device)
@@ -652,7 +651,8 @@ def run_training(args, model, **kwargs):
                     raise NotImplementedError
 
                 # Inference Step 3.2: inference, batch_samples shape: batch, <samples>, c, t, h, w
-                # no backprop_mode=args.backprop_mode because it is inference process 
+                # no backprop_mode=args.backprop_mode because it is inference process
+                seed_everything_self(args.seed)
                 if isinstance(peft_model, torch.nn.parallel.DistributedDataParallel):
                     batch_samples = batch_ddim_sampling(peft_model.module, cond, noise_shape, args.n_samples, \
                                                         args.ddim_steps, args.ddim_eta, args.unconditional_guidance_scale, None, decode_frame=args.decode_frame, **kwargs)