Update app.py

app.py

@@ -1,8 +1,35 @@
+import gradio as gr
+from transformers import ViltProcessor, ViltForQuestionAnswering
+import torch
 
-from transformers import pipeline
+torch.hub.download_url_to_file('https://cocodataset.org/#explore?id=531313', 'dog.jpg')
+torch.hub.download_url_to_file('https://cdn.openai.com/dall-e-2/demos/text2im/astronaut/horse/photo/0.jpg', 'astronaut.jpg')
 
-image_to_text = pipeline("image-to-text", model="nlpconnect/vit-gpt2-image-captioning")
+def getAnswer(query, image):
+    encoding = processor(image, text, return_tensors="pt")
+    
+    # forward pass
+    with torch.no_grad():
+     outputs = model(**encoding)
+     
+    logits = outputs.logits
+    idx = logits.argmax(-1).item()
+    predicted_answer = model.config.id2label[idx]
+   
+    return predicted_answer
 
-image_to_text("https://ankur3107.github.io/assets/images/image-captioning-example.png")
+image = gr.inputs.Image(type="pil")
+question = gr.inputs.Textbox(label="Question about the image")
+answer = gr.outputs.Textbox(label="Predicted answer")
+examples = [["dog.jpg", "Where is the dog lying?"], ["astronaut.jpg", "What's the astronaut riding on?"]]
 
-# [{'generated_text': 'a soccer game with a player jumping to catch the ball '}]
+
+title="Visual question and answering"
+
+interface = gr.Interface(fn=getAnswer, 
+                         inputs=[image, question], 
+                         outputs=answer, 
+                         examples=examples, 
+                         title=title,
+                         enable_queue=True)
+iface.launch()