Upload 5 files

app.py

@@ -0,0 +1,48 @@
+import gradio as gr
+from tasks.image_caption import image_captioning
+from tasks.image_retrieval import image_retrieval
+from tasks.visual_qa import visual_qa
+
+caption_interface = gr.Interface(
+    fn=image_captioning,
+    inputs=gr.Image(type="pil", label="Upload Image"),
+    outputs=gr.Textbox(label="Generated Caption"),
+    title="Image Captioning",
+    description="Generate a caption for the uploaded image.",
+    allow_flagging="never"  
+)
+
+retrieval_interface = gr.Interface(
+    fn=image_retrieval,
+    inputs=[
+        gr.Textbox(label="Image URL"),
+        gr.Textbox(label="Description Text")
+    ],
+    outputs=[
+        gr.Image(label="Retrieved Image"),  
+        gr.Textbox(label="Matching Probability")
+    ],
+    title="Image Retrieval",
+    description="Check if the image and text match semantically.",
+    allow_flagging="never"  
+)
+
+vqa_interface = gr.Interface(
+    fn=visual_qa,
+    inputs=[
+        gr.Image(type="pil", label="Upload Image"),
+        gr.Textbox(label="Question")
+    ],
+    outputs=gr.Textbox(label="Answer"),
+    title="Visual Question Answering",
+    description="Answer questions about the uploaded image.",
+    allow_flagging="never"  
+)
+
+# Combine vision-langauge tasks into a tabbed interface
+app = gr.TabbedInterface(
+    interface_list=[caption_interface, retrieval_interface, vqa_interface],
+    tab_names=["Image Captioning", "Image Retrieval", "Visual Q&A"]
+)
+
+app.launch()

requirements.txt

@@ -0,0 +1,5 @@
+transformers
+gradio
+torch
+requests
+pillow

tasks/image_caption.py

@@ -0,0 +1,10 @@
+from transformers import AutoProcessor, BlipForConditionalGeneration
+
+caption_id = "Salesforce/blip-image-captioning-base"
+caption_model = BlipForConditionalGeneration.from_pretrained(caption_id)
+caption_processor = AutoProcessor.from_pretrained(caption_id)
+
+def image_captioning(image):
+    inputs = caption_processor(image, "a photograph of", return_tensors="pt")
+    out = caption_model.generate(**inputs)
+    return caption_processor.decode(out[0], skip_special_tokens=True)

tasks/image_retrieval.py

@@ -0,0 +1,21 @@
+from transformers import AutoProcessor, BlipForImageTextRetrieval
+from PIL import Image
+import requests
+import torch
+
+retrieval_id = "Salesforce/blip-itm-base-coco"
+retrieval_model = BlipForImageTextRetrieval.from_pretrained(retrieval_id)
+retrieval_processor = AutoProcessor.from_pretrained(retrieval_id)
+
+def image_retrieval(image_url, text):
+    try:
+        raw_image = Image.open(requests.get(image_url, stream=True).raw).convert('RGB')
+        inputs = retrieval_processor(images=raw_image, text=text, return_tensors="pt")
+        itm_scores = retrieval_model(**inputs)[0]
+        itm_score = torch.nn.functional.softmax(itm_scores, dim=1)
+        probability = itm_score[0][1].item()
+        
+        return raw_image, f"The image and text are matched with a probability of {probability:.4f}"
+    except Exception as e:
+        return None, f"Error: {str(e)}"
+        

tasks/visual_qa.py

@@ -0,0 +1,10 @@
+from transformers import AutoProcessor, BlipForQuestionAnswering
+
+vqa_id = "Salesforce/blip-vqa-base"
+vqa_model = BlipForQuestionAnswering.from_pretrained(vqa_id)
+vqa_processor = AutoProcessor.from_pretrained(vqa_id)
+
+def visual_qa(image, question):
+    inputs = vqa_processor(image, question, return_tensors="pt")
+    out = vqa_model.generate(**inputs)
+    return vqa_processor.decode(out[0], skip_special_tokens=True)