reworked inpaint/mask, color guide, explainations

app.py

@@ -13,6 +13,11 @@ class Config:
 
 config = Config()
 
+def update_mask_editor(img):
+    if img['background'] is None:
+        return None
+    return create_padded_image(img)
+
 def create_advanced_options():
     
     negative_text = gr.Textbox(label="Negative Prompt", placeholder="Describe what not to include (1-1024 characters)", max_lines=1)
@@ -40,6 +45,7 @@ with gr.Blocks() as demo:
             justify-content: center !important;
             width: 100% !important;
         }
+        
     </style>
     """)
     gr.Markdown("<h1>Amazon Nova Canvas Image Generation</h1>", elem_classes="center-markdown" )
@@ -47,10 +53,8 @@ with gr.Blocks() as demo:
     with gr.Tab("Text to Image"):
         with gr.Column():
             gr.Markdown("""
-            <div style="text-align: center;">
-                        Generate an image from a text prompt using the Amazon Nova Canvas model.
-             </div>
-            """)
+                Generate an image from a text prompt using the Amazon Nova Canvas model.
+            """, elem_classes="center-markdown")
             prompt = gr.Textbox(label="Prompt", placeholder="Enter a text prompt (1-1024 characters)", max_lines=4)
             gr.Button("Generate Prompt").click(generate_nova_prompt, outputs=prompt)
             error_box = gr.Markdown(visible=False, label="Error", elem_classes="center-markdown")
@@ -62,58 +66,69 @@ with gr.Blocks() as demo:
     with gr.Tab("Inpainting"):
         with gr.Column():
             gr.Markdown("""
-            <div style="text-align: center;">
-                Modify specific areas of your image using inpainting. Upload your image and choose one of two ways to specify the areas you want to edit: 
-                You can use a photo editing tool to draw masks (using pure black for areas to edit and pure white for areas to preserve) or 
-                use the Mask Prompt field to direct the model in how to infer the mask.  Create an optional prompt to tell the model how to fill in the area you mask.
-            </div>
-            """)
-            image = gr.Image(type='pil', label="Input Image")
-            with gr.Accordion("Optional Prompt", open=False):
-                prompt = gr.Textbox(label="Prompt", placeholder="Describe what to generate (1-1024 characters)", max_lines=4)
-                gr.Button("Generate Prompt").click(generate_nova_prompt, outputs=prompt)
-            mask_prompt = gr.Textbox(label="Mask Prompt", placeholder="Describe regions to edit", max_lines=1)
-            with gr.Accordion("Mask Image", open=False):
-                mask_image = gr.Image(type='pil', label="Mask Image")
+            
+                Modify specific areas of your image using inpainting. Upload your base Image then choose one of two ways to specify the areas you want to edit: 
+                You can use the in app editing tool to draw masks for areas to edit or use the Mask Prompt field to direct the model how to infer the mask.  <b>ONLY 
+                ONE</b> of these methods can be used at a time.  Create an optional prompt to tell the model how to fill in the area you mask.
+           
+            """, elem_classes="center-markdown")
+            mask_image = gr.ImageEditor(
+                type="pil",
+                height="100%",
+                width="100%",
+                crop_size="1:1",
+                brush={"color": "#000000", "radius": 25},
+                show_download_button=False,
+                show_share_button=False,
+                label="Draw mask (black areas will be edited)",
+            )
+            with gr.Accordion("Optional Mask Prompt", open=False):
+                mask_prompt = gr.Textbox(label="Mask Prompt", placeholder="Describe regions to edit", max_lines=1)
+            prompt = gr.Textbox(label="Optional Prompt", placeholder="Describe what to generate (1-1024 characters) in the masked area", max_lines=4)
+            gr.Button("Generate Prompt").click(generate_nova_prompt, outputs=prompt)
             error_box = gr.Markdown(visible=False, label="Error", elem_classes="center-markdown")
             output = gr.Image()
             with gr.Accordion("Advanced Options", open=False):
                 negative_text, width, height, quality, cfg_scale, seed = create_advanced_options()
             
-            gr.Button("Generate").click(inpainting, inputs=[image, mask_prompt, mask_image, prompt, negative_text, height, width, quality, cfg_scale, seed], outputs=[output, error_box])
+            gr.Button("Generate").click(inpainting, inputs=[mask_image,mask_prompt, prompt, negative_text, height, width, quality, cfg_scale, seed], outputs=[output, error_box])
 
     with gr.Tab("Outpainting"):
         with gr.Column():
             gr.Markdown("""
-            <div style="text-align: center;">
-                Modify areas outside of your image using outpainting. The image you upload should have some transparency around the edges that you'd like to outpaint
-                into.  In options, you can choose to precisley follow the mask or transition smoothly between the masked area and the non-masked area. Choose one of two ways to specify the areas you want to edit: 
-                You can use a photo editing tool to draw masks extended outside of an images original borders (using pure black for areas to edit and pure 
-                white for areas to preserve) or use the Mask Prompt field to direct the model in how to infer the mask. Create an optional prompt to tell the model how to fill in the area you mask.
-            </div>
-            """)
-            image = gr.Image(type='pil', label="Input Image")
-            with gr.Accordion("Optional Prompt", open=False):
-                prompt = gr.Textbox(label="Prompt", placeholder="Describe what to generate (1-1024 characters)", max_lines=4)
-                gr.Button("Generate Prompt").click(generate_nova_prompt, outputs=prompt)
-            mask_prompt = gr.Textbox(label="Mask Prompt", placeholder="Describe regions to edit", max_lines=1)
-            with gr.Accordion("Mask Image", open=False):
-                mask_image = gr.Image(type='pil', label="Mask Image")
+                Modify areas outside of your image using outpainting. Give the image a transparent border by adding padding then draw
+                a mask on the image or border where you would like the model to generate new content.  The other option is to allow the model to infer the mask from the Mask Prompt.  In options, you can choose to precisley follow the mask or transition smoothly 
+                between the masked area and the non-masked area.  Create an optional prompt to tell the model how to fill in the area you mask.
+            """, elem_classes="center-markdown")
+            mask_image = gr.ImageEditor(
+                type="pil",
+                height="100%",
+                width="100%",
+                crop_size="1:1",
+                brush={"color": "#000000", "radius": 25},
+                show_download_button=False,
+                show_share_button=False,
+                label="Draw mask (black areas will be edited)"
+            )
+            gr.Button("Create Padding").click(fn=update_mask_editor, inputs=[mask_image], outputs=[mask_image])
+            
+            with gr.Accordion("Optional Mask Prompt", open=False):
+                mask_prompt = gr.Textbox(label="Mask Prompt", placeholder="Describe regions to edit", max_lines=1)
+            prompt = gr.Textbox(label="Prompt", placeholder="Describe what to generate (1-1024 characters)", max_lines=4)
+            gr.Button("Generate Prompt").click(generate_nova_prompt, outputs=prompt)
             error_box = gr.Markdown(visible=False, label="Error", elem_classes="center-markdown")
             output = gr.Image()
             with gr.Accordion("Advanced Options", open=False):
                 outpainting_mode = gr.Radio(choices=["DEFAULT", "PRECISE"], value="DEFAULT", label="Outpainting Mode")
                 negative_text, width, height, quality, cfg_scale, seed = create_advanced_options()
             
-            gr.Button("Generate").click(outpainting, inputs=[image, mask_prompt, mask_image, prompt, negative_text, outpainting_mode, height, width, quality, cfg_scale, seed], outputs=[output, error_box])
+            gr.Button("Generate").click(outpainting, inputs=[mask_image, mask_prompt, prompt, negative_text, outpainting_mode, height, width, quality, cfg_scale, seed], outputs=[output, error_box])
 
     with gr.Tab("Image Variation"):
         with gr.Column():
             gr.Markdown("""
-            <div style="text-align: center;">
-                Create a variation image based on up to 5 other images and a Similarity slider available in options.  You can add a prompt to direct the model (optional).  Images should be .png or .jpg.
-                </div>
-            """)
+                Create a variation image based on up to 5 other images and a Similarity slider available in options.  You can add a prompt to direct the model (optional).  Images should be .png or .jpg.  
+            """, elem_classes="center-markdown")
             images = gr.File(type='filepath', label="Input Images", file_count="multiple", file_types=["image"])
             with gr.Accordion("Optional Prompt", open=False):
                 prompt = gr.Textbox(label="Prompt", placeholder="Enter a text prompt (1-1024 characters)", max_lines=4)
@@ -129,12 +144,10 @@ with gr.Blocks() as demo:
     with gr.Tab("Image Conditioning"):
         with gr.Column():
             gr.Markdown("""
-            <div style="text-align: center;">
                 Generate an image conditioned by an input image.  You need to add a text prompt to direct the model (required).
                 You have two modes to control the conditioning,"CANNY" and "SEGMENTATION".  CANNY will follow the edges of the conditioning image closely.
                 SEGMENTATION will follow the layout or shapes of the conditioning image. 
-                </div>
-            """)
+            """, elem_classes="center-markdown")
             condition_image = gr.Image(type='pil', label="Condition Image")
             prompt = gr.Textbox(label="Prompt", placeholder="Enter a text prompt (1-1024 characters)", max_lines=4)
             gr.Button("Generate Prompt").click(generate_nova_prompt, outputs=prompt)
@@ -149,14 +162,16 @@ with gr.Blocks() as demo:
     with gr.Tab("Color Guided"):
         with gr.Column():
             gr.Markdown("""
-            <div style="text-align: center;">
-                Generate an image using a color palette.  You must include a text prompt if you choose to include an image, the subject and style will be used as a reference. 
-                The colors of the image will also be incorporated, along with the colors from the colors list. A color list is also required if you choose not to include one,
-                a generic list has been provided.
-                </div>
-            """)
-              
-            colors = gr.Textbox(label="Colors", placeholder="Enter up to 10 colors as hex values, e.g., #00FF00,#FCF2AB", max_lines=1)
+                Generate an image using a color palette.  This mode requires a text prompt and a color list.  If you choose to include an image, the subject and style will be used as a reference. 
+                The colors of the image will also be incorporated, along with the colors from the colors list. A generic color list has been provided behind the scenes if one isn't added.
+            """, elem_classes="center-markdown")
+            with gr.Row():
+                with gr.Column(scale=75): 
+                    colors = gr.Textbox(label="Colors", placeholder="Enter up to 10 colors as hex values, e.g., #00FF00,#FCF2AB", max_lines=1)
+                with gr.Column(scale=25):  
+                    color_picker = gr.ColorPicker(label="Color Picker", show_label=False)
+                    #add_color_button = gr.Button("Add Color")  Work out Color Picker Collapsing and Rerendering
+                    #add_color_button.click(fn=add_color_to_list, inputs=[colors, color_picker], outputs=colors)
             prompt = gr.Textbox(label="Text", placeholder="Enter a text prompt (1-1024 characters)", max_lines=4)
             gr.Button("Generate Prompt").click(generate_nova_prompt, outputs=prompt)
             with gr.Accordion("Optional Reference Image", open=False):
@@ -170,10 +185,8 @@ with gr.Blocks() as demo:
     with gr.Tab("Background Removal"):
         with gr.Column():
             gr.Markdown("""
-            <div style="text-align: center;">
                 Remove the background from an image.
-                </div>
-            """)
+            """, elem_classes="center-markdown")
             image = gr.Image(type='pil', label="Input Image")
             error_box = gr.Markdown(visible=False, label="Error", elem_classes="center-markdown")
             output = gr.Image()

functions.py

@@ -4,64 +4,88 @@ import random
 import gradio as gr
 from PIL import Image
 from generate import *
+import numpy as np
 from typing import Dict, Any
 from processImage import process_and_encode_image
 
-def display_image(image_bytes):
-    if isinstance(image_bytes, str):
-        # If we received a string (error message), return it to be displayed
-        return None, gr.update(visible=True, value=image_bytes)
-    elif image_bytes:
-        # If we received image bytes, process and display the image
-        return Image.open(io.BytesIO(image_bytes)), gr.update(visible=False)
+def rgba_to_hex(rgba):
+    r, g, b, _ = [int(float(x)) for x in rgba[5:-1].split(',')]
+    return f"#{r:02X}{g:02X}{b:02X}"
+
+def add_color_to_list(current_colors, new_color):
+    new_color_hex = rgba_to_hex(new_color)
+    color_list = current_colors.split(',')
+    if new_color_hex not in color_list and len(color_list) < 10:
+        color_list.append(new_color_hex)
+    return ','.join(filter(None, color_list))
+
+def create_padded_image(image, padding_percent=100):
+    image = image['background']
+    if image.mode != 'RGBA':
+        image = image.convert('RGBA')
+    
+    width, height = image.size
+    new_width = int(width * (1 + padding_percent/100))
+    new_height = int(height * (1 + padding_percent/100))
+    
+    padded = Image.new('RGBA', (new_width, new_height), (0, 0, 0, 0))  
+    
+    x_offset = (new_width - width) // 2
+    y_offset = (new_height - height) // 2
+    
+    padded.paste(image, (x_offset, y_offset))
+    return padded
+
+def process_composite_to_mask(original_image, composite_image, transparent=False):
+    original_array = np.array(original_image.convert('RGBA'))
+    if transparent:
+        white_background = Image.new('RGBA', original_image.size, (255, 255, 255, 255))
+        white_background.paste(original_image, (0, 0), original_image)
+        return white_background
+    if composite_image is None:
+        mask = np.full(original_array.shape[:2], 255, dtype=np.uint8)  # Start with white
+        transparent_areas = original_array[:, :, 3] == 0  # Alpha channel is 0 for transparent pixels
+        mask[transparent_areas] = 0
     else:
-        # Handle None case
-        return None, gr.update(visible=False)
-
-def process_optional_params(**kwargs) -> Dict[str, Any]:
-    return {k: v for k, v in kwargs.items() if v is not None}
-
-def process_images(primary=None, secondary=None, validate=True) -> Dict[str, str]:
-    if validate and primary is None:
-        raise ValueError("Primary image is required.")
-    result = {}
-    if primary:
-        result["image"] = process_and_encode_image(primary)
-    if secondary:
-        result["maskImage"] = process_and_encode_image(secondary)
-    return result
-
-def create_image_generation_config(height=1024, width=1024, quality="standard", cfg_scale=8.0, seed=0):
-    return {
-        "numberOfImages": 1,
-        "height": height,
-        "width": width,
-        "quality": quality,
-        "cfgScale": cfg_scale,
-        "seed": seed
-    }
+        composite_array = np.array(composite_image.convert('RGBA'))
+    
+        difference = np.any(original_array != composite_array, axis=2)
+        mask = np.full(original_array.shape[:2], 255, dtype=np.uint8)
+        mask[difference] = 0
+        
+        
+        
+    
+    return Image.fromarray(mask, mode='L')
 
 def build_request(task_type, params, height=1024, width=1024, quality="standard", cfg_scale=8.0, seed=0):
-    param_dict = {"TEXT_IMAGE": "textToImageParams", "INPAINTING": "inPaintingParams", 
-                  "OUTPAINTING":"outPaintingParams","IMAGE_VARIATION":"imageVariationParams",
-                  "COLOR_GUIDED_GENERATION":"colorGuidedGenerationParams","BACKGROUND_REMOVAL":"backgroundRemovalParams"}
+    param_dict = {
+        "TEXT_IMAGE": "textToImageParams",
+        "INPAINTING": "inPaintingParams", 
+        "OUTPAINTING": "outPaintingParams",
+        "IMAGE_VARIATION": "imageVariationParams",
+        "COLOR_GUIDED_GENERATION": "colorGuidedGenerationParams",
+        "BACKGROUND_REMOVAL": "backgroundRemovalParams"
+    }
+    
     return json.dumps({
         "taskType": task_type,
         param_dict[task_type]: params,
-        "imageGenerationConfig": create_image_generation_config(
-            height=height,
-            width=width,
-            quality=quality,
-            cfg_scale=cfg_scale,
-            seed=seed
-        )
+        "imageGenerationConfig": {
+            "numberOfImages": 1,
+            "height": height,
+            "width": width,
+            "quality": quality,
+            "cfgScale": cfg_scale,
+            "seed": seed
+        }
     })
 
 def check_return(result):
     if not isinstance(result, bytes):
         return None, gr.update(visible=True, value=result)
-        
-    return Image.open(io.BytesIO(result)), gr.update(visible=False)
+    
+    return Image.open(io.BytesIO(result)), gr.update(value=None,visible=False)
 
 
 def text_to_image(prompt, negative_text=None, height=1024, width=1024, quality="standard", cfg_scale=8.0, seed=0):
@@ -74,21 +98,23 @@ def text_to_image(prompt, negative_text=None, height=1024, width=1024, quality="
     return check_return(result)
     
 
-def inpainting(image, mask_prompt=None, mask_image=None, text=None, negative_text=None, height=1024, width=1024, quality="standard", cfg_scale=8.0, seed=0):
-    images = process_images(primary=image, secondary=None)
+def inpainting(mask_image, mask_prompt=None, text=None, negative_text=None, height=1024, width=1024, quality="standard", cfg_scale=8.0, seed=0):
+    image = process_and_encode_image(mask_image['background'])
+    if len(image) < 200:
+        return None, gr.update(visible=True, value=image)
     
-    for value in images.values():
-        if len(value) < 200:
-            return None, gr.update(visible=True, value=value)
-    # Prepare the inPaintingParams dictionary
     if mask_prompt and mask_image:
         raise ValueError("You must specify either maskPrompt or maskImage, but not both.")
     if not mask_prompt and not mask_image:
         raise ValueError("You must specify either maskPrompt or maskImage.")
-
-    # Prepare the inPaintingParams dictionary with the appropriate mask parameter
+    
+    if mask_image and 'composite' in mask_image:
+        mask = process_composite_to_mask(mask_image['background'], mask_image['composite'])
+        mask_image = process_and_encode_image(mask)
+    
     in_painting_params = {
-        **images,  # Unpacks image and maskImage if present
+        "image": image,  
+        **({"maskImage": mask_image} if mask_image not in [None, ""] else {}),
         **({"maskPrompt": mask_prompt} if mask_prompt not in [None, ""] else {}),
         **({"text": text} if text not in [None, ""] else {}),
         **({"negativeText": negative_text} if negative_text not in [None, ""] else {})
@@ -99,25 +125,29 @@ def inpainting(image, mask_prompt=None, mask_image=None, text=None, negative_tex
     
     return check_return(result)
 
-def outpainting(image, mask_prompt=None, mask_image=None, text=None, negative_text=None, outpainting_mode="DEFAULT", height=1024, width=1024, quality="standard", cfg_scale=8.0, seed=0):
-    images = process_images(primary=image, secondary=None)
-    for value in images.values():
-        if len(value) < 200:
-            return None, gr.update(visible=True, value=value)
+def outpainting(mask_image, mask_prompt=None, text=None, negative_text=None, outpainting_mode="DEFAULT", height=1024, width=1024, quality="standard", cfg_scale=8.0, seed=0):
+    image = process_and_encode_image(mask_image['background'])
+    if len(value) < 200:
+        return None, gr.update(visible=True, value=value)
 
     if mask_prompt and mask_image:
         raise ValueError("You must specify either maskPrompt or maskImage, but not both.")
     if not mask_prompt and not mask_image:
         raise ValueError("You must specify either maskPrompt or maskImage.")
+    
+    if mask_image and 'composite' in mask_image:
+        mask = process_composite_to_mask(mask_image['background'], None)
+        image = process_composite_to_mask(mask_image['background'], None, True)
+        image = process_and_encode_image(image)
+        mask_image = process_and_encode_image(mask)
 
     # Prepare the outPaintingParams dictionary
     out_painting_params = {
-        **images,  # Unpacks image and maskImage if present
-        **process_optional_params(
-            **({"maskPrompt": mask_prompt} if mask_prompt  not in [None, ""] else {}),
-            **({"text": text} if text  not in [None, ""] else {}),
-            **({"negativeText": negative_text} if negative_text not in [None, ""] else {})
-        )
+        "image": image,
+        **({"maskImage": mask_image} if mask_image not in [None, ""] else {}),
+        **({"maskPrompt": mask_prompt} if mask_prompt not in [None, ""] else {}),
+        **({"text": text} if text not in [None, ""] else {"text": " "}),
+        **({"negativeText": negative_text} if negative_text not in [None, ""] else {})
     }
 
     body = build_request("OUTPAINTING", out_painting_params, height, width, quality, cfg_scale, seed)
@@ -138,6 +168,7 @@ def image_variation(images, text=None, negative_text=None, similarity_strength=0
     # Prepare the imageVariationParams dictionary
     image_variation_params = {
         "images": encoded_images,
+        **({"similarityStrength": similarity_strength} if similarity_strength not in [None, ""] else {}),
         **({"text": text} if text not in [None, ""] else {}),
         **({"negativeText": negative_text} if negative_text not in [None, ""] else {})
     }
@@ -148,16 +179,16 @@ def image_variation(images, text=None, negative_text=None, similarity_strength=0
     return check_return(result)
 
 def image_conditioning(condition_image, text, negative_text=None, control_mode="CANNY_EDGE", control_strength=0.7, height=1024, width=1024, quality="standard", cfg_scale=8.0, seed=0):
-    condition_image_encoded = process_images(primary=condition_image)
-    for value in condition_image_encoded.values():
-        if len(value) < 200:
-            return None, gr.update(visible=True, value=value)
+    condition_image_encoded = process_and_encode_image(condition_image)
+    
+    if len(condition_image_encoded) < 200:
+        return None, gr.update(visible=True, value=condition_image_encoded)
     # Prepare the textToImageParams dictionary
     text_to_image_params = {
         "text": text,
         "controlMode": control_mode,
         "controlStrength": control_strength,
-        "conditionImage": condition_image_encoded.get('image'),
+        "conditionImage": condition_image_encoded,
         **({"negativeText": negative_text} if negative_text not in [None, ""] else {})
     }
     body = build_request("TEXT_IMAGE", text_to_image_params, height, width, quality, cfg_scale, seed)
@@ -169,19 +200,18 @@ def color_guided_content(text=None, reference_image=None, negative_text=None, co
     reference_image_str = None
 
     if reference_image is not None and not isinstance(reference_image, type(None)):
-       
-        reference_image_encoded = process_images(primary=reference_image)
-        for value in reference_image_encoded.values():
-            if len(value) < 200:
-                return None, gr.update(visible=True, value=value)
-            reference_image_str = reference_image_encoded.get('image')
+        reference_image_encoded = process_and_encode_image(reference_image)
+        
+        if len(reference_image_encoded) < 200:
+            return None, gr.update(visible=True, value=reference_image_encoded)
+            
     if not colors:
         colors = "#FF5733,#33FF57,#3357FF,#FF33A1,#33FFF5,#FF8C33,#8C33FF,#33FF8C,#FF3333,#33A1FF"
     
     color_guided_generation_params = {
         "text": text,
-        "colors": colors.split(','),
-        **({"referenceImage": reference_image_str} if reference_image_str is not None else {}),    
+        "colors": [color.strip() for color in colors.split(',')],
+        **({"referenceImage": reference_image_encoded} if reference_image_str is not None else {}),    
         **({"negativeText": negative_text} if negative_text not in [None, ""] else {})
     }
 
@@ -191,15 +221,15 @@ def color_guided_content(text=None, reference_image=None, negative_text=None, co
     return check_return(result)
 
 def background_removal(image):
-    input_image = process_images(primary=image)
-    for value in input_image.values():
-        if len(value) < 200:
-            return None, gr.update(visible=True, value=value)
+    input_image = process_and_encode_image(image)
+    
+    if len(input_image) < 200:
+        return None, gr.update(visible=True, value=input_image)
         
     body = json.dumps({
         "taskType": "BACKGROUND_REMOVAL",
         "backgroundRemovalParams": {
-            "image": input_image.get('image')  
+            "image": input_image
         }
     })
     result = generate_image(body)

generate.py

@@ -173,7 +173,7 @@ def check_rate_limit(body):
             raise ImageError(rate_limit_message.format('Premium'))
         rate_data['premium'].append(current_time)
     else:  # standard
-        if len(rate_data['standard']) >= 6:
+        if len(rate_data['standard']) >= 100:
             raise ImageError(rate_limit_message.format('Standard'))
         rate_data['standard'].append(current_time)