update chat

app.py

@@ -207,11 +207,12 @@ def gradio_reset(chat_state, img_list):
         value="Upload & Start Chat", interactive=True), chat_state, img_list
 
 
-def upload_img(gr_img, text_input, chat_state):
+def upload_img(gr_img, text_input, chat_state,chatbot):
     if gr_img is None:
         return None, None, gr.update(interactive=True), chat_state, None
     chat_state = []
     img_list = []
+    chatbot = chatbot + [[gr_img, None]]
     llm_message = chat.upload_img(gr_img, chat_state, img_list)
     return gr.update(interactive=False), gr.update(interactive=True, placeholder='Type and press Enter'), gr.update(
         value="Start Chatting", interactive=False), chat_state, img_list
@@ -221,6 +222,7 @@ def gradio_ask(user_message, chatbot, chat_state):
     if len(user_message) == 0:
         return gr.update(interactive=True, placeholder='Input should not be empty!'), chatbot, chat_state
 
+
     chat.ask(user_message, chat_state)
     chatbot = chatbot + [[user_message, None]]
     return '', chatbot, chat_state
@@ -271,7 +273,7 @@ with gr.Blocks() as demo:
             chatbot = gr.Chatbot(label='Compositional-VLM')
             text_input = gr.Textbox(label='User', placeholder='Please upload your image first', interactive=False)
 
-    upload_button.click(upload_img, [image, text_input, chat_state],
+    upload_button.click(upload_img, [image, text_input, chat_state,chatbot],
                         [image, text_input, upload_button, chat_state, img_list])
 
     text_input.submit(gradio_ask, [text_input, chatbot, chat_state], [text_input, chatbot, chat_state]).then(

multimodal/open_flamingo/chat/conversation.py

@@ -317,14 +317,15 @@ class Chat:
         visual_token_id = self.tokenizer("<|#visual#|>", add_special_tokens=False)["input_ids"][-1]
         previsual_token_id = self.tokenizer("<|#previsual#|>", add_special_tokens=False)["input_ids"][-1]
         prebox_token_id = self.tokenizer("<|#prebox#|>", add_special_tokens=False)["input_ids"][-1]
-        size = self.vis_processor.size["shortest_edge"]
+        size = 224
         model.eval()
         # "/gpfs/u/home/LMCG/LMCGljnn/scratch-shared/cdl/tmp_img/chat_vis/chat19.png"
-        image_path = input("Please enter the image path: ")
-        image = Image.open(image_path).convert("RGB")
+        # image_path = input("Please enter the image path: ")
+        image = img_list[0].convert("RGB")
         image = image.resize((size, size))
         print(f"image size: {image.size}")
-        batch_images = preprocess_image(img_list[0], self.vis_processor).unsqueeze(0).unsqueeze(1).unsqueeze(0).to("cuda")
+        batch_images = preprocess_image(image, self.vis_processor).unsqueeze(0).unsqueeze(1).unsqueeze(0)
+        
         # conversation = []
         human_sentence = None
         conv.append({