trying to mirror what is here

https://huggingface.co/spaces/nielsr/comparing-captioning-models/blob/main/app.py

app.py

@@ -3,14 +3,15 @@ import torch
 from PIL import Image
 from lavis.models import load_model_and_preprocess
 from lavis.processors import load_processor
-from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoFeatureExtractor
 
 # Load model and preprocessors for Image-Text Matching (LAVIS)
 device = torch.device("cuda") if torch.cuda.is_available() else "cpu"
 model_itm, vis_processors, text_processors = load_model_and_preprocess("blip2_image_text_matching", "pretrain", device=device, is_eval=True)
 
-# Load tokenizer and model for Image Captioning (TextCaps)
+# Load tokenizer, feature extractor, and model for Image Captioning (TextCaps)
 tokenizer_caption = AutoTokenizer.from_pretrained("microsoft/git-large-r-textcaps")
+feature_extractor = AutoFeatureExtractor.from_pretrained("microsoft/git-large-r-textcaps")
 model_caption = AutoModelForCausalLM.from_pretrained("microsoft/git-large-r-textcaps").to(device)
 
 # List of statements for Image-Text Matching
@@ -26,7 +27,9 @@ statements = [
 
 # Function to generate image captions using TextCaps
 def generate_image_captions(image):
-    inputs = tokenizer_caption(image, return_tensors="pt", padding=True, truncation=True).to(device)
+    # Preprocess image using feature extractor
+    inputs = feature_extractor(images=image, return_tensors="pt", padding=True, truncation=True).to(device)
+    # Generate captions
     outputs = model_caption.generate(**inputs)
     caption = tokenizer_caption.decode(outputs[0], skip_special_tokens=True)
     return caption
@@ -64,4 +67,3 @@ output = gr.outputs.Textbox(label="Results")
 
 iface = gr.Interface(fn=process_images_and_statements, inputs=image_input, outputs=output, title="Image Captioning and Image-Text Matching")
 iface.launch()
-