Upload app2_blip.py

app2_blip.py

@@ -0,0 +1,97 @@
+import streamlit as st
+import gc
+from PIL import Image
+import torch
+from transformers import (
+    BlipProcessor,
+    BlipForConditionalGeneration,
+    DetrImageProcessor,
+    DetrForObjectDetection,
+)
+import google.generativeai as genai
+
+# Configure Generative AI
+genai.configure(api_key='AIzaSyB1wxTDQcB2YT_6l2nm4MrhAmCVPzfkHNU')
+gemini_model = genai.GenerativeModel("gemini-1.5-flash")
+
+# Load BLIP model and processor
+@st.cache_resource
+def load_blip():
+    processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
+    model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
+    return processor, model
+
+# Load DETR model and processor
+@st.cache_resource
+def load_detr():
+    processor = DetrImageProcessor.from_pretrained("facebook/detr-resnet-50", revision="no_timm")
+    model = DetrForObjectDetection.from_pretrained("facebook/detr-resnet-50", revision="no_timm")
+    return processor, model
+
+blip_processor, blip_model = load_blip()
+detr_processor, detr_model = load_detr()
+
+# Streamlit app
+st.title("Art Of Visual Storytelling")
+
+# Dropdown menu for genre selection
+genre = st.selectbox(
+    "Select the genre of the story:",
+    ["Fantasy", "Adventure", "Sci-Fi", "Romance", "Mystery", "Horror", "Comedy", "Drama"]
+)
+
+# Slider for specifying story length
+story_length = st.slider(
+    "Select the desired story length (number of words):",
+    min_value=50, max_value=1000, value=200, step=50
+)
+
+# Image upload and processing
+uploaded_image = st.file_uploader("Upload an Image", type=['png', 'jpg', 'jpeg'])
+
+if uploaded_image:
+    # Display the uploaded image
+    raw_image = Image.open(uploaded_image).convert('RGB')
+    st.image(raw_image, caption='Uploaded Image', use_column_width=True)
+
+    # Detect objects using DETR model
+    detr_inputs = detr_processor(images=raw_image, return_tensors="pt")
+    detr_model.eval()
+    with torch.no_grad():
+        detr_outputs = detr_model(**detr_inputs)
+
+    target_sizes = torch.tensor([raw_image.size[::-1]])  # (height, width)
+    results = detr_processor.post_process_object_detection(detr_outputs, target_sizes=target_sizes, threshold=0.9)[0]
+
+    # Extract object names
+    detected_objects = []
+    for score, label in zip(results["scores"], results["labels"]):
+        label_name = detr_model.config.id2label[label.item()]
+        detected_objects.append(label_name)
+
+    # Display the detected objects before generating the caption
+    st.subheader("Detected Objects:")
+    st.text(", ".join(set(detected_objects)))  # Show unique objects
+
+    # Generate caption using BLIP model
+    blip_inputs = blip_processor(raw_image, return_tensors="pt")
+    blip_outputs = blip_model.generate(**blip_inputs)
+    caption = blip_processor.decode(blip_outputs[0], skip_special_tokens=True)
+
+    # Display the generated caption after detected objects
+    st.subheader("Generated Caption:")
+    st.text(caption)
+
+    # Submit button to generate the story
+    if st.button("Generate Story"):
+        prompt = (
+            f"I have an image. The caption is '{caption}', and the detected objects are {', '.join(set(detected_objects))}. "
+            f"Write a {genre.lower()} story with approximately {story_length} words using elements of this caption and the detected objects."
+        )
+        response = gemini_model.generate_content(prompt)
+        st.subheader("Generated Story:")
+        st.text_area("Story Output", value=response.text, height=300)
+
+    # Cleanup memory
+    del raw_image, detr_inputs, detr_outputs, blip_inputs, blip_outputs, results
+    gc.collect()