# Model Setting pretrained_model_name_or_path: stabilityai/stable-video-diffusion-img2vid # stabilityai/pretrained load_unet_path: ../model_weights/ROB_regular_weights/v4_VL_paper/checkpoint-99000 # None/specific path This is for pretrained-UNet path load_controlnet_path: # None/specific path For checkpoint loaded from pretrained-Controlnet Path video_seq_length: 14 process_fps: 7 train_noise_aug_strength: 0.1 scheduler: EDM conditioning_dropout_prob: 0.1 # Dataset Setting data_loader_type: thisthat # thisthat dataset_name: Bridge # Bridge dataset_path: [../sanity_check/bridge_v1_TT14, ../sanity_check/bridge_v2_TT14] # ../Bridge_filter_flow, ../Bridge_v2_filter_flow/] output_dir: checkpoints/img2video height: 256 # Ratio that is functional: 256:384 576:1024 320:448 320:576 512:640 448:640 width: 384 # It is said that the height and width should be a scale of 64 dataloader_num_workers: 4 # For Debug, it only needs 1 flip_aug_prob: 0.45 # Whether we flip the GT and cond vertically # No acceleration_tolerance, since TT dataset already filter those out # Text setting use_text: True # If this is True, we will use text value pretrained_tokenizer_name_or_path: stabilityai/stable-diffusion-2-1-base # Use SD 2.1 empty_prompts_proportion: 0.0 mix_ambiguous: False # Whether we mix ambiguous prompt for "this" and "that" # Mask setting mask_unet_vae: False # Whether we use mask to map latents to be zero padding mask_controlnet_vae: False mask_proportion: 0.0 # Condition Setting conditioning_channels: 3 # Usually it is 3 num_points_left: # 1 # For flow: You can only choose one between flow_select_rate and num_points_left; num_points_left should be higher priority flow_select_rate: 0.99 # For flow threshold_factor: 0.2 # For flow dilate: True # Traj must be True for dilate inner_conditioning_scale: 1.0 # Conditioning scale for the internal value, defauly is starting from 1.0 outer_conditioning_scale: 1.0 # Outer Conditioning Scale for whole conditioning trainable copy 这里有点意思，直接不小心设定成2.0了 # Motion setting motion_bucket_id: 200 dataset_motion_mean: 25 # For 14 fps, it is N(25, 10) dataset_motion_std: 10 # For 25 fps, it is N(18, 7) svd_motion_mean: 180 svd_motion_std: 30 # Training setting resume_from_checkpoint: False # latest/False num_train_iters: 30100 # Will automatically choose the checkpoints partial_finetune: False # Whether we just tune some params to speed up train_batch_size: 1 # This is the batch size per GPU checkpointing_steps: 3000 validation_step: 300 logging_name: logging seed: 42 validation_img_folder: datasets/validation_TT14 validation_store_folder: validation_videos checkpoints_total_limit: 15 # Noise Strength noise_mean: 0.5 # Regular Img2Video: (0.7, 1.6); Text2Video: (0.5, 1.4) noise_std: 1.4 # Inference num_inference_steps: 25 use_instructpix2pix: False # Whether we will use the instructPix2Pix mode, which involves 3 inputs; it may needs tuning to have better result at the end. inference_noise_aug_strength: 0.1 inference_max_guidance_scale: 3.0 # Take training and testing at different scenario inference_guess_mode: False # Whether we use guess mode in the contorlnet image_guidance_scale: 2.5 # Empirically, 2.5 is the best value Seems not using this now # Learning Rate and Optimizer learning_rate: 5e-6 # 5e-6 is the LR we test that is just right scale_lr: False # TODO: Is it needed to scale the learning rate? adam_beta1: 0.9 adam_beta2: 0.999 use_8bit_adam: True # Need this to save more memory adam_weight_decay: 1e-2 adam_epsilon: 1e-08 lr_warmup_steps: 500 lr_decay_scale: 0.5 # Other Setting mixed_precision: fp16 gradient_accumulation_steps: 1 # ???? gradient_checkpointing: 1 # ???? report_to: tensorboard