GUI

Experiments/Baseline/GUI.py

@@ -1,24 +1,49 @@
 import streamlit as st
+import openai
+import json
 from PIL import Image
+from google.oauth2 import service_account
+from baseline_utils import detect_text_in_image, summarize_diary_text, analyze_writer_image, generate_comic_book
 
-# You can create a web or mobile-based GUI so that users can experience your solution. Suggested libraries include https://www.gradio.app/ or https://streamlit.io/.
-st.title('Handwritten Diary to Cartoon Book')
-uploaded_diary = st.file_uploader("Upload your diary image", type=["png", "jpg", "jpeg"])
-uploaded_writer_image = st.file_uploader("Upload your photo", type=["png", "jpg", "jpeg"])
+# Load secrets
+openai_api_key = st.secrets["general"]["openai_api_key"]
+google_service_account_info = json.loads(st.secrets["general"]["google_service_account"])
+gemini_api_key = st.secrets["general"]["gemini_api_key"]
+
+# Initialize OpenAI
+openai.api_key = openai_api_key
+
+# Function to get Google credentials
+def get_google_credentials():
+    return service_account.Credentials.from_service_account_info(google_service_account_info)
+
+st.title('Handwritten Diary to Comic Book')
+uploaded_diary = st.file_uploader("Upload your handwritten diary image", type=["png", "jpg", "jpeg"])
+uploaded_writer_image = st.file_uploader("Upload a photo of the writer", type=["png", "jpg", "jpeg"])
 
 if uploaded_diary and uploaded_writer_image:
-    st.write("Analyzing your diary...")
+    st.write("Analyzing your diary and writer...")
+
+    # Read the uploaded images
+    diary_image = Image.open(uploaded_diary)
+    writer_image = Image.open(uploaded_writer_image)
 
-    diary_text = detect_text_in_image(uploaded_diary)
-    summarized_text = summarize_diary_text(diary_text)
+    # Save uploaded images temporarily (Streamlit does this automatically with file objects)
+    diary_image_path = uploaded_diary.name
+    writer_image_path = uploaded_writer_image.name
 
+    # Detect text from the diary image
+    google_credentials = get_google_credentials()
+    detected_text = detect_text_in_image(diary_image_path, google_credentials)
+    summarized_text = summarize_diary_text(detected_text, openai_api_key)
     st.write(f"Summarized Diary Text: {summarized_text}")
 
-    writer_description = analyze_writer_image(uploaded_writer_image)
-    st.write(f"Diary Writer Description: {writer_description}")
+    # Analyze the writer's image using Gemini API
+    writer_summary = analyze_writer_image(writer_image_path, gemini_api_key)
+    st.write(f"Writer Description: {writer_summary}")
 
-    # Generate cartoon image
-    prompt = f"{summarized_text}, featuring a person who {writer_description}"
-    generated_image = generate_image(prompt)
+    # Generate the comic book based on the summaries
+    st.write("Generating comic book images...")
+    generate_comic_book(summarized_text, writer_summary, num_pages=5)
 
-    st.image(generated_image, caption="Generated Cartoon Image")
+    st.write("Comic book generated successfully!")

Experiments/Baseline/baseline.py

@@ -1,12 +1,12 @@
 from baseline_utils import *
 from keys.keys import *
 
-diary_image_path = "images/test_sample.jpeg"
-writer_image_path = "images/writer.jpg"
+diary_image_path = "input_images/test_sample.jpeg"
+writer_image_path = "input_images/writer.jpeg"
 credentials_path = "keys/service_account_credentials.json"
 
 # Detect text from the image using the provided credentials
 detected_text = detect_text_in_image(diary_image_path, credentials_path)
 diary_summary = summarize_diary_text(detected_text, open_ai_keys)
 writer_summary = analyze_writer_image(writer_image_path, gemini_keys)
-generate_image(diary_summary, writer_summary)
+generate_comic_book(diary_summary, writer_summary)

Experiments/Baseline/baseline_utils.py

@@ -5,10 +5,10 @@ import io
 import google.generativeai as genai
 from diffusers import AutoPipelineForText2Image
 import torch
-
+import os
 
 # Utilize the Google Cloud Vision API to recognize text in the
-# input images (diary images), https://cloud.google.com/vision.
+# input input_images (diary input_images), https://cloud.google.com/vision.
 def detect_text_in_image(image_path, credentials_path):
     # Load the service account key from the credentials JSON file
     credentials = service_account.Credentials.from_service_account_file(credentials_path)
@@ -66,22 +66,23 @@ def analyze_writer_image(image_path, api_key):
     model = genai.GenerativeModel("gemini-1.5-flash")
     myfile = genai.upload_file(image_path)
     result = model.generate_content(
-        [myfile, "\n\n", "Can you give a textual description of the image?"]
+        [myfile, "\n\n", "Can you give a very short description of the person in the image?"]
     )
     return result.text
 
 
 # Now that you have text from the diary and text describing the diary writer,
 # you can utilize the SDXL-Turbo stable diffusion model to generate
-# images https://huggingface.co/stabilityai/sdxl-turbo.
-# You can try to output several images for a diary entry. Analyze how accurate the results,
+# input_images https://huggingface.co/stabilityai/sdxl-turbo.
+# You can try to output several input_images for a diary entry. Analyze how accurate the results,
 # and think about what could be improved.
-def generate_image(diary_text, writer_description):
+def generate_comic_book(diary_text, writer_description, num_pages=4):
     pipe = AutoPipelineForText2Image.from_pretrained(
         "stabilityai/sdxl-turbo",
         torch_dtype=torch.float16,
         variant="fp16",
-        cache_dir="./SDXL-Turbo")
+        cache_dir="./SDXL-Turbo"
+    )
 
     # Check for available device: CUDA, MPS, or CPU
     if torch.cuda.is_available():
@@ -97,11 +98,28 @@ def generate_image(diary_text, writer_description):
     # Move the model to the selected device
     pipe = pipe.to(device)
 
-    # Generate the image with a simple prompt
-    prompt = f'Writer Description: {writer_description} \n\n Diary: {diary_text}'
-    print(prompt)
-    image = pipe(prompt=prompt, num_inference_steps=1, guidance_scale=0.0).images[0]
+    # Create a directory to store the comic book input_images
+    os.makedirs("comic_book", exist_ok=True)
+
+    # Split diary text into multiple segments/scenes for comic book pages
+    diary_scenes = diary_text.split('.')[:num_pages]  # Split by periods, limiting to `num_pages`
+
+    # Iterate over each scene, generating a page for each one
+    for i, scene in enumerate(diary_scenes):
+        prompt = (f'Comic Book Style: \n'
+                  f'Actor Description: {writer_description} \n'
+                  f'Diary Scene: {scene.strip()}\n'
+                  f'Generate an cartoon image to represent this diary scene.')
+
+        print(f"Generating comic page {i + 1} with prompt:\n{prompt}\n")
+
+        # Generate the image
+        image = pipe(prompt=prompt, num_inference_steps=30, guidance_scale=7.5).images[0]
+
+        # Save the generated image
+        image_path = f"comic_book/page_{i + 1}.png"
+        image.save(image_path)
+        print(f"Page {i + 1} saved as {image_path}")
 
-    # Save the generated image
-    image.save("generated_image.png")
+    print("Comic book generation complete!")