Upload folder using huggingface_hub

controlnet/config.json

@@ -1,7 +1,6 @@
 {
   "_class_name": "ControlNetModel",
-  "_diffusers_version": "0.28.2",
-  "_name_or_path": "lllyasviel/control_v11p_sd15_scribble",
+  "_diffusers_version": "0.23.1",
   "act_fn": "silu",
   "addition_embed_type": null,
   "addition_embed_type_num_heads": 64,
@@ -10,7 +9,6 @@
   "block_out_channels": [
     320,
     640,
-    1280,
     1280
   ],
   "class_embed_type": null,
@@ -24,10 +22,9 @@
   "controlnet_conditioning_channel_order": "rgb",
   "cross_attention_dim": 768,
   "down_block_types": [
+    "DownBlock2D",
     "CrossAttnDownBlock2D",
-    "CrossAttnDownBlock2D",
-    "CrossAttnDownBlock2D",
-    "DownBlock2D"
+    "CrossAttnDownBlock2D"
   ],
   "downsample_padding": 1,
   "encoder_hid_dim": null,
@@ -36,17 +33,20 @@
   "freq_shift": 0,
   "global_pool_conditions": false,
   "in_channels": 4,
-  "layers_per_block": 2,
+  "layers_per_block": 1,
   "mid_block_scale_factor": 1,
-  "mid_block_type": "UNetMidBlock2DCrossAttn",
   "norm_eps": 1e-05,
   "norm_num_groups": 32,
   "num_attention_heads": null,
   "num_class_embeds": null,
-  "only_cross_attention": false,
+  "only_cross_attention": [
+    true,
+    false,
+    false
+  ],
   "projection_class_embeddings_input_dim": null,
   "resnet_time_scale_shift": "default",
   "transformer_layers_per_block": 1,
   "upcast_attention": false,
   "use_linear_projection": false
-}
+}

controlnet/diffusion_pytorch_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8199ce735834688ef40e758d7e7086b69361da4bb41dbcbbdd90b0172fcae69a
-size 722598648
+oid sha256:d005da74cf9e639b3b9d6a49ff845690fd2081fead531c714c98d2694e490c96
+size 728496840

logs/text2image-fine-tune/1717891836.1011167/events.out.tfevents.1717891836.a8343c8809c1.14061.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a8e64e850b157d87f33faa2a9b2af0def9995035385e707fc97517cc1cf456c
+size 2457

logs/text2image-fine-tune/1717891836.1036482/hparams.yml

@@ -0,0 +1,47 @@
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+cache_dir: null
+center_crop: true
+checkpointing_steps: 5000
+checkpoints_total_limit: null
+dataloader_num_workers: 0
+dataset_config_name: null
+enable_xformers_memory_efficient_attention: false
+gradient_accumulation_steps: 4
+gradient_checkpointing: true
+lambda_kd_feat: 1.0
+lambda_kd_output: 1.0
+lambda_sd: 1.0
+learning_rate: 5.0e-05
+local_rank: -1
+logging_dir: logs
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1.0
+max_train_samples: null
+max_train_steps: 400000
+mixed_precision: fp16
+non_ema_revision: null
+num_train_epochs: 4939
+num_valid_images: 2
+output_dir: ./results/v2-base_kd_bk_tiny
+pretrained_model_name_or_path: stabilityai/stable-diffusion-2-1-base
+random_flip: true
+report_to: all
+resolution: 512
+resume_from_checkpoint: null
+revision: null
+scale_lr: false
+seed: 1234
+train_batch_size: 32
+train_data_dir: ./data/laion_aes/preprocessed_11k
+unet_config_name: bk_tiny
+unet_config_path: ./src/unet_config_v2-base
+use_8bit_adam: false
+use_copy_weight_from_teacher: true
+use_ema: true
+valid_prompt: a golden vase with different flowers
+valid_steps: 500

logs/text2image-fine-tune/1717891979.0476391/events.out.tfevents.1717891979.a8343c8809c1.14739.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76614d8dbc8489000da018488807c6f040af5051fb5119365dad25c8e17894a7
+size 2457

logs/text2image-fine-tune/1717891979.0580502/hparams.yml

@@ -0,0 +1,47 @@
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+cache_dir: null
+center_crop: true
+checkpointing_steps: 5000
+checkpoints_total_limit: null
+dataloader_num_workers: 0
+dataset_config_name: null
+enable_xformers_memory_efficient_attention: false
+gradient_accumulation_steps: 4
+gradient_checkpointing: true
+lambda_kd_feat: 1.0
+lambda_kd_output: 1.0
+lambda_sd: 1.0
+learning_rate: 5.0e-05
+local_rank: -1
+logging_dir: logs
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1.0
+max_train_samples: null
+max_train_steps: 0
+mixed_precision: fp16
+non_ema_revision: null
+num_train_epochs: 0
+num_valid_images: 2
+output_dir: ./results/v2-base_kd_bk_tiny
+pretrained_model_name_or_path: stabilityai/stable-diffusion-2-1-base
+random_flip: true
+report_to: all
+resolution: 512
+resume_from_checkpoint: null
+revision: null
+scale_lr: false
+seed: 1234
+train_batch_size: 32
+train_data_dir: ./data/laion_aes/preprocessed_11k
+unet_config_name: bk_tiny
+unet_config_path: ./src/unet_config_v2-base
+use_8bit_adam: false
+use_copy_weight_from_teacher: true
+use_ema: true
+valid_prompt: a golden vase with different flowers
+valid_steps: 500

logs/text2image-fine-tune/1717893073.4145217/events.out.tfevents.1717893073.a8343c8809c1.18650.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0125e74a77defb57f223e6550fd2d98a7f5cbe4b98ebf1bd191ae3def7cd417
+size 2457

logs/text2image-fine-tune/1717893073.4179482/hparams.yml

@@ -0,0 +1,47 @@
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+cache_dir: null
+center_crop: true
+checkpointing_steps: 5000
+checkpoints_total_limit: null
+dataloader_num_workers: 0
+dataset_config_name: null
+enable_xformers_memory_efficient_attention: false
+gradient_accumulation_steps: 4
+gradient_checkpointing: true
+lambda_kd_feat: 1.0
+lambda_kd_output: 1.0
+lambda_sd: 1.0
+learning_rate: 5.0e-05
+local_rank: -1
+logging_dir: logs
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1.0
+max_train_samples: null
+max_train_steps: 0
+mixed_precision: fp16
+non_ema_revision: null
+num_train_epochs: 0
+num_valid_images: 2
+output_dir: ./results/v2-base_kd_bk_tiny
+pretrained_model_name_or_path: stabilityai/stable-diffusion-2-1-base
+random_flip: true
+report_to: all
+resolution: 512
+resume_from_checkpoint: null
+revision: null
+scale_lr: false
+seed: 1234
+train_batch_size: 32
+train_data_dir: ./data/laion_aes/preprocessed_11k
+unet_config_name: bk_tiny
+unet_config_path: ./src/unet_config_v2-base
+use_8bit_adam: false
+use_copy_weight_from_teacher: true
+use_ema: true
+valid_prompt: a golden vase with different flowers
+valid_steps: 500

logs/text2image-fine-tune/1717893880.4456894/events.out.tfevents.1717893880.a8343c8809c1.22493.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2b6011f7bcca594668a35d88f35e1e3197d1f44c1a6415e9aecf13905ce52ff
+size 2457

logs/text2image-fine-tune/1717893880.4486566/hparams.yml

@@ -0,0 +1,47 @@
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+cache_dir: null
+center_crop: true
+checkpointing_steps: 5000
+checkpoints_total_limit: null
+dataloader_num_workers: 0
+dataset_config_name: null
+enable_xformers_memory_efficient_attention: false
+gradient_accumulation_steps: 4
+gradient_checkpointing: true
+lambda_kd_feat: 1.0
+lambda_kd_output: 1.0
+lambda_sd: 1.0
+learning_rate: 5.0e-05
+local_rank: -1
+logging_dir: logs
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1.0
+max_train_samples: null
+max_train_steps: 0
+mixed_precision: fp16
+non_ema_revision: null
+num_train_epochs: 0
+num_valid_images: 2
+output_dir: ./results/v2-base_kd_bk_tiny
+pretrained_model_name_or_path: stabilityai/stable-diffusion-2-1-base
+random_flip: true
+report_to: all
+resolution: 512
+resume_from_checkpoint: null
+revision: null
+scale_lr: false
+seed: 1234
+train_batch_size: 32
+train_data_dir: ./data/laion_aes/preprocessed_11k
+unet_config_name: bk_tiny
+unet_config_path: ./src/unet_config_v2-base
+use_8bit_adam: false
+use_copy_weight_from_teacher: true
+use_ema: true
+valid_prompt: a golden vase with different flowers
+valid_steps: 500

logs/text2image-fine-tune/1717895383.2177856/events.out.tfevents.1717895383.a8343c8809c1.28476.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8eeedce53c0773941d57e221739c105f5ab5364fba7eca7fd0aee99419e50dd1
+size 2457

logs/text2image-fine-tune/1717895383.2211077/hparams.yml

@@ -0,0 +1,47 @@
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+adam_weight_decay: 0.01
+allow_tf32: false
+cache_dir: null
+center_crop: true
+checkpointing_steps: 5000
+checkpoints_total_limit: null
+dataloader_num_workers: 0
+dataset_config_name: null
+enable_xformers_memory_efficient_attention: false
+gradient_accumulation_steps: 4
+gradient_checkpointing: true
+lambda_kd_feat: 1.0
+lambda_kd_output: 1.0
+lambda_sd: 1.0
+learning_rate: 5.0e-05
+local_rank: -1
+logging_dir: logs
+lr_scheduler: constant
+lr_warmup_steps: 0
+max_grad_norm: 1.0
+max_train_samples: null
+max_train_steps: 0
+mixed_precision: fp16
+non_ema_revision: null
+num_train_epochs: 0
+num_valid_images: 2
+output_dir: ./results/v2-base_kd_bk_tiny
+pretrained_model_name_or_path: stabilityai/stable-diffusion-2-1-base
+random_flip: true
+report_to: all
+resolution: 512
+resume_from_checkpoint: null
+revision: null
+scale_lr: false
+seed: 1234
+train_batch_size: 32
+train_data_dir: ./data/laion_aes/preprocessed_11k
+unet_config_name: bk_tiny
+unet_config_path: ./src/unet_config_v2-base
+use_8bit_adam: false
+use_copy_weight_from_teacher: true
+use_ema: true
+valid_prompt: a golden vase with different flowers
+valid_steps: 500

logs/text2image-fine-tune/events.out.tfevents.1717891826.a8343c8809c1.14061.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0993749572b8d4492840b40a252b58bc6fff959df7b7b5612ce4b26c8fa9b3be
+size 88

logs/text2image-fine-tune/events.out.tfevents.1717891969.a8343c8809c1.14739.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:334b181de4118f862b5697938b2a1b7dd558db7d878569dedbee12829fa3e6a1
+size 88

logs/text2image-fine-tune/events.out.tfevents.1717892979.a8343c8809c1.18650.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03f3f85998300f5d293fa468aa5c0bac1b5bb1fc4876f2babf54d4fbd502e2f0
+size 88

logs/text2image-fine-tune/events.out.tfevents.1717893862.a8343c8809c1.22493.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:285e14b72f5161fbe9a5220697a36f984882ae3f91cce389f20df1a272e84f4c
+size 88

logs/text2image-fine-tune/events.out.tfevents.1717895332.a8343c8809c1.28476.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a7c252deb57685f5a199756c29586144c15cded161eba117c8796a0d6c12782
+size 88

model_index.json

@@ -1,7 +1,7 @@
 {
   "_class_name": "StableDiffusionControlNetPipeline",
   "_diffusers_version": "0.28.2",
-  "_name_or_path": "CompVis/stable-diffusion-v1-4",
+  "_name_or_path": "stabilityai/stable-diffusion-2-1-base",
   "controlnet": [
     "diffusers",
     "ControlNetModel"
@@ -14,10 +14,10 @@
     null,
     null
   ],
-  "requires_safety_checker": true,
+  "requires_safety_checker": false,
   "safety_checker": [
-    "stable_diffusion",
-    "StableDiffusionSafetyChecker"
+    null,
+    null
   ],
   "scheduler": [
     "diffusers",

text_encoder/config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "CompVis/stable-diffusion-v1-4",
+  "_name_or_path": "stabilityai/stable-diffusion-2-1-base",
   "architectures": [
     "CLIPTextModel"
   ],
@@ -7,16 +7,16 @@
   "bos_token_id": 0,
   "dropout": 0.0,
   "eos_token_id": 2,
-  "hidden_act": "quick_gelu",
-  "hidden_size": 768,
+  "hidden_act": "gelu",
+  "hidden_size": 1024,
   "initializer_factor": 1.0,
   "initializer_range": 0.02,
-  "intermediate_size": 3072,
+  "intermediate_size": 4096,
   "layer_norm_eps": 1e-05,
   "max_position_embeddings": 77,
   "model_type": "clip_text_model",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 12,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 23,
   "pad_token_id": 1,
   "projection_dim": 512,
   "torch_dtype": "float16",

text_encoder/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77795e2023adcf39bc29a884661950380bd093cf0750a966d473d1718dc9ef4e
-size 246144864
+oid sha256:681c555376658c81dc273f2d737a2aeb23ddb6d1d8e5b3a7064636d359a22668
+size 680821096

tokenizer/special_tokens_map.json

@@ -13,7 +13,7 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|endoftext|>",
+  "pad_token": "!",
   "unk_token": {
     "content": "<|endoftext|>",
     "lstrip": false,

unet/config.json

@@ -5,12 +5,15 @@
   "addition_embed_type": null,
   "addition_embed_type_num_heads": 64,
   "addition_time_embed_dim": null,
-  "attention_head_dim": 8,
+  "attention_head_dim": [
+    5,
+    10,
+    20
+  ],
   "attention_type": "default",
   "block_out_channels": [
     320,
     640,
-    1280,
     1280
   ],
   "center_input_sample": false,
@@ -18,13 +21,12 @@
   "class_embeddings_concat": false,
   "conv_in_kernel": 3,
   "conv_out_kernel": 3,
-  "cross_attention_dim": 768,
+  "cross_attention_dim": 1024,
   "cross_attention_norm": null,
   "down_block_types": [
     "CrossAttnDownBlock2D",
     "CrossAttnDownBlock2D",
-    "CrossAttnDownBlock2D",
-    "DownBlock2D"
+    "CrossAttnDownBlock2D"
   ],
   "downsample_padding": 1,
   "dropout": 0.0,
@@ -57,11 +59,10 @@
   "timestep_post_act": null,
   "transformer_layers_per_block": 1,
   "up_block_types": [
-    "UpBlock2D",
     "CrossAttnUpBlock2D",
     "CrossAttnUpBlock2D",
     "CrossAttnUpBlock2D"
   ],
   "upcast_attention": false,
-  "use_linear_projection": false
+  "use_linear_projection": true
 }

unet/diffusion_pytorch_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9e9d51e8c2e414f5e4ef31164276798dcad2761559e290ed2c80d9900fe7c77
-size 1929435608
+oid sha256:1381a43284ede537a46fbfb17380eb80c5bbed310fee24620064d22917690bbd
+size 1307346120

vae/config.json

@@ -1,7 +1,7 @@
 {
   "_class_name": "AutoencoderKL",
   "_diffusers_version": "0.28.2",
-  "_name_or_path": "CompVis/stable-diffusion-v1-4",
+  "_name_or_path": "stabilityai/stable-diffusion-2-1-base",
   "act_fn": "silu",
   "block_out_channels": [
     128,
@@ -23,7 +23,7 @@
   "layers_per_block": 2,
   "norm_num_groups": 32,
   "out_channels": 3,
-  "sample_size": 512,
+  "sample_size": 768,
   "scaling_factor": 0.18215,
   "up_block_types": [
     "UpDecoderBlock2D",

vae/diffusion_pytorch_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4fbcf0ebe55a0984f5a5e00d8c4521d52359af7229bb4d81890039d2aa16dd7c
+oid sha256:3e4c08995484ee61270175e9e7a072b66a6e4eeb5f0c266667fe1f45b90daf9a
 size 167335342