Update app.py

app.py

@@ -3,13 +3,14 @@ import torch
 from PIL import Image
 from lavis.models import load_model_and_preprocess
 from lavis.processors import load_processor
-from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import ViTFeatureExtractor, AutoTokenizer, AutoModelForCausalLM
 
 # Load model and preprocessors for Image-Text Matching (LAVIS)
 device = torch.device("cuda") if torch.cuda.is_available() else "cpu"
 model_itm, vis_processors, text_processors = load_model_and_preprocess("blip2_image_text_matching", "pretrain", device=device, is_eval=True)
 
-# Load tokenizer and model for Image Captioning (TextCaps)
+# Load feature extractor, tokenizer, and model for Image Captioning (TextCaps)
+feature_extractor = ViTFeatureExtractor.from_pretrained("microsoft/git-large-r-textcaps")
 tokenizer_caption = AutoTokenizer.from_pretrained("microsoft/git-large-r-textcaps")
 model_caption = AutoModelForCausalLM.from_pretrained("microsoft/git-large-r-textcaps").to(device)
 
@@ -26,7 +27,9 @@ statements = [
 
 # Function to generate image captions using TextCaps
 def generate_image_captions(image):
-    inputs = tokenizer_caption(image, return_tensors="pt", padding=True, truncation=True).to(device)
+    # Process the image using feature_extractor
+    inputs = feature_extractor(images=image, return_tensors="pt", padding=True, truncation=True).to(device)
+    # Generate captions
     outputs = model_caption.generate(**inputs)
     caption = tokenizer_caption.decode(outputs[0], skip_special_tokens=True)
     return caption
@@ -60,7 +63,7 @@ def process_images_and_statements(image):
 
 # Gradio interface
 image_input = gr.inputs.Image()
-output = gr.outputs.Textbox(label="Results")
+output = gr.outputs.Textbox(label="Output")
 
-iface = gr.Interface(fn=process_images_and_statements, inputs=image_input, outputs=output, title="Image Captioning and Image-Text Matching")
-iface.launch()
+iface = gr.Interface(fn=process_images_and_statements, inputs=image_input, outputs=output)
+iface.launch()