Add autocast

app.py

@@ -88,7 +88,7 @@ def get_layout_image(response):
 def get_layout_image_gallery(response):
     return [get_layout_image(response)]
 
-def get_ours_image(response, seed, num_inference_steps=20, dpm_scheduler=True, overall_prompt_override="", fg_seed_start=20, fg_blending_ratio=0.1, frozen_step_ratio=0.4, gligen_scheduled_sampling_beta=0.3, so_negative_prompt=DEFAULT_SO_NEGATIVE_PROMPT, overall_negative_prompt=DEFAULT_OVERALL_NEGATIVE_PROMPT, show_so_imgs=False, scale_boxes=False):
+def get_ours_image(response, seed, num_inference_steps=20, dpm_scheduler=True, use_autocast=False, overall_prompt_override="", fg_seed_start=20, fg_blending_ratio=0.1, frozen_step_ratio=0.4, gligen_scheduled_sampling_beta=0.3, so_negative_prompt=DEFAULT_SO_NEGATIVE_PROMPT, overall_negative_prompt=DEFAULT_OVERALL_NEGATIVE_PROMPT, show_so_imgs=False, scale_boxes=False):
     if response == "":
         response = layout_placeholder
     gen_boxes, bg_prompt = parse_input(response)
@@ -107,7 +107,7 @@ def get_ours_image(response, seed, num_inference_steps=20, dpm_scheduler=True, o
         
     image_np, so_img_list = run_ours(
         spec, bg_seed=seed, overall_prompt_override=overall_prompt_override, fg_seed_start=fg_seed_start, 
-        fg_blending_ratio=fg_blending_ratio,frozen_step_ratio=frozen_step_ratio, 
+        fg_blending_ratio=fg_blending_ratio,frozen_step_ratio=frozen_step_ratio, use_autocast=use_autocast,
         gligen_scheduled_sampling_beta=gligen_scheduled_sampling_beta, num_inference_steps=num_inference_steps, scheduler_key=scheduler_key,
         so_negative_prompt=so_negative_prompt, overall_negative_prompt=overall_negative_prompt
     )
@@ -242,7 +242,8 @@ with gr.Blocks(
                     frozen_step_ratio = gr.Slider(0, 1, value=0.4, step=0.1, label="Foreground frozen steps ratio (higher: preserve object attributes; lower: higher coherence; set to 0: (almost) equivalent to vanilla GLIGEN except details)")
                     gligen_scheduled_sampling_beta = gr.Slider(0, 1, value=0.3, step=0.1, label="GLIGEN guidance steps ratio (the beta value)")
                     num_inference_steps = gr.Slider(1, 50, value=20, step=1, label="Number of inference steps")
-                    dpm_scheduler = gr.Checkbox(label="Use DPM scheduler (unchecked: DDIM scheduler, may have better coherence, recommend 50 inference steps)", show_label=False, value=True)
+                    dpm_scheduler = gr.Checkbox(label="Use DPM scheduler (unchecked: DDIM scheduler, may have better coherence, recommend 50 or even more inference steps)", show_label=False, value=True)
+                    use_autocast = gr.Checkbox(label="Use FP16 Mixed Precision", show_label=False, value=True)
                     fg_seed_start = gr.Slider(0, 10000, value=20, step=1, label="Seed for foreground variation")
                     fg_blending_ratio = gr.Slider(0, 1, value=0.1, step=0.01, label="Variations added to foreground for single object generation (0: no variation, 1: max variation)")
                     so_negative_prompt = gr.Textbox(lines=1, label="Negative prompt for single object generation", value=DEFAULT_SO_NEGATIVE_PROMPT)
@@ -256,7 +257,7 @@ with gr.Blocks(
                     label="Generated image", show_label=False, elem_id="gallery", columns=[1], rows=[1], object_fit="contain", preview=True
                 )
         visualize_btn.click(fn=get_layout_image_gallery, inputs=response, outputs=gallery, api_name="visualize-layout")
-        generate_btn.click(fn=get_ours_image, inputs=[response, seed, num_inference_steps, dpm_scheduler, overall_prompt_override, fg_seed_start, fg_blending_ratio, frozen_step_ratio, gligen_scheduled_sampling_beta, so_negative_prompt, overall_negative_prompt, show_so_imgs, scale_boxes], outputs=gallery, api_name="layout-to-image")
+        generate_btn.click(fn=get_ours_image, inputs=[response, seed, num_inference_steps, dpm_scheduler, use_autocast, overall_prompt_override, fg_seed_start, fg_blending_ratio, frozen_step_ratio, gligen_scheduled_sampling_beta, so_negative_prompt, overall_negative_prompt, show_so_imgs, scale_boxes], outputs=gallery, api_name="layout-to-image")
 
         gr.Examples(
             examples=stage2_examples,

generation.py

@@ -76,8 +76,8 @@ def get_masked_latents_all_list(so_prompt_phrase_word_box_list, input_latents_li
 
 def run(
     spec, bg_seed = 1, overall_prompt_override="", fg_seed_start = 20, frozen_step_ratio=0.4, gligen_scheduled_sampling_beta = 0.3, num_inference_steps = 20,
-    so_center_box = False, fg_blending_ratio = 0.1, scheduler_key='dpm_scheduler', so_negative_prompt = DEFAULT_SO_NEGATIVE_PROMPT, overall_negative_prompt = DEFAULT_OVERALL_NEGATIVE_PROMPT, so_horizontal_center_only = True,
-    align_with_overall_bboxes = False, horizontal_shift_only = True
+    so_center_box = False, fg_blending_ratio = 0.1, scheduler_key='dpm_scheduler', so_negative_prompt = DEFAULT_SO_NEGATIVE_PROMPT, overall_negative_prompt = DEFAULT_OVERALL_NEGATIVE_PROMPT, so_horizontal_center_only = True, 
+    align_with_overall_bboxes = False, horizontal_shift_only = True, use_autocast = False
 ):
     """    
     so_center_box: using centered box in single object generation
@@ -111,62 +111,59 @@ def run(
         discourage_mask_below_confidence=discourage_mask_below_confidence, discourage_mask_below_coarse_iou=discourage_mask_below_coarse_iou,
         height=height, width=width, H=H, W=W
     )
-
     
     # Note that so and overall use different negative prompts
 
-    so_prompts = [item[0] for item in so_prompt_phrase_word_box_list]
-    if so_prompts:
-        so_input_embeddings = models.encode_prompts(prompts=so_prompts, tokenizer=tokenizer, text_encoder=text_encoder, negative_prompt=so_negative_prompt, one_uncond_input_only=True)
-    else:
-        so_input_embeddings = []
-
-    overall_input_embeddings = models.encode_prompts(prompts=[overall_prompt], tokenizer=tokenizer, negative_prompt=overall_negative_prompt, text_encoder=text_encoder)
-
+    with torch.autocast("cuda", enabled=use_autocast):
+        so_prompts = [item[0] for item in so_prompt_phrase_word_box_list]
+        if so_prompts:
+            so_input_embeddings = models.encode_prompts(prompts=so_prompts, tokenizer=tokenizer, text_encoder=text_encoder, negative_prompt=so_negative_prompt, one_uncond_input_only=True)
+        else:
+            so_input_embeddings = []
 
+        overall_input_embeddings = models.encode_prompts(prompts=[overall_prompt], tokenizer=tokenizer, negative_prompt=overall_negative_prompt, text_encoder=text_encoder)
+        
+        input_latents_list, latents_bg = latents.get_input_latents_list(
+            model_dict, bg_seed=bg_seed, fg_seed_start=fg_seed_start, 
+            so_boxes=so_boxes, fg_blending_ratio=fg_blending_ratio, height=height, width=width, verbose=False
+        )
+        latents_all_list, mask_tensor_list, so_img_list = get_masked_latents_all_list(
+            so_prompt_phrase_word_box_list, input_latents_list, 
+            gligen_scheduled_sampling_beta=gligen_scheduled_sampling_beta,
+            sam_refine_kwargs=sam_refine_kwargs, so_input_embeddings=so_input_embeddings, num_inference_steps=num_inference_steps, scheduler_key=scheduler_key, verbose=verbose
+        )
 
-    
-    input_latents_list, latents_bg = latents.get_input_latents_list(
-        model_dict, bg_seed=bg_seed, fg_seed_start=fg_seed_start, 
-        so_boxes=so_boxes, fg_blending_ratio=fg_blending_ratio, height=height, width=width, verbose=False
-    )
-    latents_all_list, mask_tensor_list, so_img_list = get_masked_latents_all_list(
-        so_prompt_phrase_word_box_list, input_latents_list, 
-        gligen_scheduled_sampling_beta=gligen_scheduled_sampling_beta,
-        sam_refine_kwargs=sam_refine_kwargs, so_input_embeddings=so_input_embeddings, num_inference_steps=num_inference_steps, scheduler_key=scheduler_key, verbose=verbose
-    )
-
-    
+        
 
-    composed_latents, foreground_indices, offset_list = latents.compose_latents_with_alignment(
-        model_dict, latents_all_list, mask_tensor_list, num_inference_steps, 
-        overall_batch_size, height, width, latents_bg=latents_bg, 
-        align_with_overall_bboxes=align_with_overall_bboxes, overall_bboxes=overall_bboxes,
-        horizontal_shift_only=horizontal_shift_only
-    )
-    
-    overall_bboxes_flattened, overall_phrases_flattened = [], []
-    for overall_bboxes_item, overall_phrase in zip(overall_bboxes, overall_phrases):
-        for overall_bbox in overall_bboxes_item:
-            overall_bboxes_flattened.append(overall_bbox)
-            overall_phrases_flattened.append(overall_phrase)
+        composed_latents, foreground_indices, offset_list = latents.compose_latents_with_alignment(
+            model_dict, latents_all_list, mask_tensor_list, num_inference_steps, 
+            overall_batch_size, height, width, latents_bg=latents_bg, 
+            align_with_overall_bboxes=align_with_overall_bboxes, overall_bboxes=overall_bboxes,
+            horizontal_shift_only=horizontal_shift_only
+        )
+        
+        overall_bboxes_flattened, overall_phrases_flattened = [], []
+        for overall_bboxes_item, overall_phrase in zip(overall_bboxes, overall_phrases):
+            for overall_bbox in overall_bboxes_item:
+                overall_bboxes_flattened.append(overall_bbox)
+                overall_phrases_flattened.append(overall_phrase)
 
-    # Generate with composed latents
+        # Generate with composed latents
 
-    # Foreground should be frozen
-    frozen_mask = foreground_indices != 0
-    
-    regen_latents, images = pipelines.generate_gligen(
-        model_dict, composed_latents, overall_input_embeddings, num_inference_steps, 
-        overall_bboxes_flattened, overall_phrases_flattened, guidance_scale=guidance_scale,
-        gligen_scheduled_sampling_beta=gligen_scheduled_sampling_beta,
-        frozen_steps=frozen_steps, frozen_mask=frozen_mask, scheduler_key=scheduler_key
-    )
+        # Foreground should be frozen
+        frozen_mask = foreground_indices != 0
+        
+        regen_latents, images = pipelines.generate_gligen(
+            model_dict, composed_latents, overall_input_embeddings, num_inference_steps, 
+            overall_bboxes_flattened, overall_phrases_flattened, guidance_scale=guidance_scale,
+            gligen_scheduled_sampling_beta=gligen_scheduled_sampling_beta,
+            frozen_steps=frozen_steps, frozen_mask=frozen_mask, scheduler_key=scheduler_key
+        )
 
-    print(f"Generation with spatial guidance from input latents and first {frozen_steps} steps frozen (directly from the composed latents input)")
-    print("Generation from composed latents (with semantic guidance)")
+        print(f"Generation with spatial guidance from input latents and first {frozen_steps} steps frozen (directly from the composed latents input)")
+        print("Generation from composed latents (with semantic guidance)")
 
-    # display(Image.fromarray(images[0]), "img", run_ind)
-    
+        # display(Image.fromarray(images[0]), "img", run_ind)
+        
     return images[0], so_img_list
 

gradio_cached_examples/39/Generated image/32ac0e0e-135a-404c-a1ee-53fdbc919db6/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/39/Generated image/32ac0e0e-135a-404c-a1ee-53fdbc919db6/6e96ae22067936bf00a4f9f9415775f561fd0152/image.png": null}

gradio_cached_examples/39/Generated image/5541f42f-a5c4-4c90-ae9c-389d0f0ea11a/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/39/Generated image/5541f42f-a5c4-4c90-ae9c-389d0f0ea11a/e8933d4d2aff4203da4600fd6eb763a04c8667ff/image.png": null}

gradio_cached_examples/39/Generated image/7dbf49b5-a987-4285-9ecb-899fc0897489/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/39/Generated image/7dbf49b5-a987-4285-9ecb-899fc0897489/1ba27d75ea6c232428e503a0336d8eb3c346c0b3/image.png": null}

gradio_cached_examples/39/Generated image/ae08bef2-f889-441a-ba1e-026445bb386a/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/39/Generated image/ae08bef2-f889-441a-ba1e-026445bb386a/1a312139177423e79631a7bf40aa1ac531efb744/image.png": null}

gradio_cached_examples/39/Generated image/d216beac-010e-4466-856c-9d92e471654c/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/39/Generated image/d216beac-010e-4466-856c-9d92e471654c/90a51edff815fd0aaef1864d6784583e800be8d8/image.png": null}

gradio_cached_examples/39/log.csv

@@ -1,6 +0,0 @@
-Generated image,flag,username,timestamp
-./gradio_cached_examples/39/Generated image/ae08bef2-f889-441a-ba1e-026445bb386a,,,2023-06-19 12:19:24.628285
-./gradio_cached_examples/39/Generated image/7dbf49b5-a987-4285-9ecb-899fc0897489,,,2023-06-19 12:19:29.717383
-./gradio_cached_examples/39/Generated image/d216beac-010e-4466-856c-9d92e471654c,,,2023-06-19 12:19:36.564223
-./gradio_cached_examples/39/Generated image/5541f42f-a5c4-4c90-ae9c-389d0f0ea11a,,,2023-06-19 12:19:39.911724
-./gradio_cached_examples/39/Generated image/32ac0e0e-135a-404c-a1ee-53fdbc919db6,,,2023-06-19 12:19:44.983434

gradio_cached_examples/40/Generated image/2b8c8fa6-273e-4993-b984-ca766b7fa093/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/40/Generated image/2b8c8fa6-273e-4993-b984-ca766b7fa093/6b8454f7801357c4c41076b4281ff36cc3c168d1/image.png": null}

gradio_cached_examples/40/Generated image/5cc41c19-e968-478e-b144-5b8ba3d07be1/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/40/Generated image/5cc41c19-e968-478e-b144-5b8ba3d07be1/65c5ef22af85300a0da31b44b1ad679a47eecc25/image.png": null}

gradio_cached_examples/40/Generated image/cd999256-9a24-4a5b-a897-5c5fa56c5a67/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/40/Generated image/cd999256-9a24-4a5b-a897-5c5fa56c5a67/101f36d55ac3f9ebcf7ff5647da0d7acb4994c97/image.png": null}

gradio_cached_examples/40/Generated image/e5bbe48b-6f7d-45c9-b75f-38f1709e0026/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/40/Generated image/e5bbe48b-6f7d-45c9-b75f-38f1709e0026/210e77149171a376e7be7a733564930e356849db/image.png": null}

gradio_cached_examples/40/Generated image/eb50d204-9a80-4e66-82a4-b24be17a05c2/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/40/Generated image/eb50d204-9a80-4e66-82a4-b24be17a05c2/44f995a0ec0c935f5881edd8af7f10e857e1a968/image.png": null}

gradio_cached_examples/40/log.csv

@@ -0,0 +1,6 @@
+Generated image,flag,username,timestamp
+./gradio_cached_examples/40/Generated image/5cc41c19-e968-478e-b144-5b8ba3d07be1,,,2023-06-20 07:17:18.077779
+./gradio_cached_examples/40/Generated image/2b8c8fa6-273e-4993-b984-ca766b7fa093,,,2023-06-20 07:17:23.197795
+./gradio_cached_examples/40/Generated image/cd999256-9a24-4a5b-a897-5c5fa56c5a67,,,2023-06-20 07:17:30.064619
+./gradio_cached_examples/40/Generated image/eb50d204-9a80-4e66-82a4-b24be17a05c2,,,2023-06-20 07:17:33.414443
+./gradio_cached_examples/40/Generated image/e5bbe48b-6f7d-45c9-b75f-38f1709e0026,,,2023-06-20 07:17:38.498364

gradio_cached_examples/49/Generated image/569b2539-1b09-422e-8f04-28e85cb5ce6b/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/49/Generated image/569b2539-1b09-422e-8f04-28e85cb5ce6b/79b47dee4bf06f02baaddf31631dadf4f0a77b1b/image.png": null}

gradio_cached_examples/49/Generated image/7ca4de19-dacd-433a-9bda-44a30411773a/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/49/Generated image/7ca4de19-dacd-433a-9bda-44a30411773a/da41c41cef06d8895f87bd51bccacb9e5ee6fc13/image.png": null}

gradio_cached_examples/49/Generated image/9d74cf63-2741-4aa1-9b9d-284ce36b1272/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/49/Generated image/9d74cf63-2741-4aa1-9b9d-284ce36b1272/916b46e1b9e7e59a0f42ea2e0e9d3ac2077ddb29/image.png": null}

gradio_cached_examples/49/Generated image/d1cff19c-eda7-411a-97bd-598780ee1514/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/49/Generated image/d1cff19c-eda7-411a-97bd-598780ee1514/111213a2bec11fbeb98d5cf421ff3f1e90ac2a6f/image.png": null}

gradio_cached_examples/49/Generated image/ff249b87-f078-4ed7-b702-d9c026c2ae0b/captions.json

@@ -1 +0,0 @@
-{"./gradio_cached_examples/49/Generated image/ff249b87-f078-4ed7-b702-d9c026c2ae0b/30ac54337ceb5917e94befaaa6939bdb2970ea50/image.png": null}

gradio_cached_examples/49/log.csv

@@ -1,6 +0,0 @@
-Generated image,flag,username,timestamp
-./gradio_cached_examples/49/Generated image/d1cff19c-eda7-411a-97bd-598780ee1514,,,2023-06-19 12:19:46.344457
-./gradio_cached_examples/49/Generated image/7ca4de19-dacd-433a-9bda-44a30411773a,,,2023-06-19 12:19:47.718673
-./gradio_cached_examples/49/Generated image/569b2539-1b09-422e-8f04-28e85cb5ce6b,,,2023-06-19 12:19:49.113759
-./gradio_cached_examples/49/Generated image/9d74cf63-2741-4aa1-9b9d-284ce36b1272,,,2023-06-19 12:19:50.442599
-./gradio_cached_examples/49/Generated image/ff249b87-f078-4ed7-b702-d9c026c2ae0b,,,2023-06-19 12:19:51.819811

gradio_cached_examples/50/Generated image/2b20f451-c8d6-4f26-b580-85a54989b5e7/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/2b20f451-c8d6-4f26-b580-85a54989b5e7/e73c626c4dd21a7ad80767910261adf9442952fc/image.png": null}

gradio_cached_examples/50/Generated image/3ef81752-e136-466d-b5d0-fa2ddec37971/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/3ef81752-e136-466d-b5d0-fa2ddec37971/6863bcdf1ddd78ec6b38cb7f7e54dfd02042cfc8/image.png": null}

gradio_cached_examples/50/Generated image/460ea7e5-92af-4b13-9c5d-1386164789d7/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/460ea7e5-92af-4b13-9c5d-1386164789d7/c881ad29f622a8f1d9a88854d93ec05142de78c0/image.png": null}

gradio_cached_examples/50/Generated image/8c451715-7470-46fc-9634-e16f88e98e49/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/8c451715-7470-46fc-9634-e16f88e98e49/5107323699a702f3dbb8cac7480802bd1ce00aa8/image.png": null}

gradio_cached_examples/50/Generated image/bbaa5527-7cf2-47e4-aa03-6d68ecbaa65b/captions.json

@@ -0,0 +1 @@
+{"./gradio_cached_examples/50/Generated image/bbaa5527-7cf2-47e4-aa03-6d68ecbaa65b/62563c47f44f246d810891c182da583bab12f8a0/image.png": null}

gradio_cached_examples/50/log.csv

@@ -0,0 +1,6 @@
+Generated image,flag,username,timestamp
+./gradio_cached_examples/50/Generated image/3ef81752-e136-466d-b5d0-fa2ddec37971,,,2023-06-20 07:17:39.853483
+./gradio_cached_examples/50/Generated image/460ea7e5-92af-4b13-9c5d-1386164789d7,,,2023-06-20 07:17:41.227531
+./gradio_cached_examples/50/Generated image/bbaa5527-7cf2-47e4-aa03-6d68ecbaa65b,,,2023-06-20 07:17:42.621290
+./gradio_cached_examples/50/Generated image/2b20f451-c8d6-4f26-b580-85a54989b5e7,,,2023-06-20 07:17:43.974086
+./gradio_cached_examples/50/Generated image/8c451715-7470-46fc-9634-e16f88e98e49,,,2023-06-20 07:17:45.349694