updates to image logging

app.py

@@ -57,7 +57,7 @@ class SaveRelevanceCallback(FlaggingCallback):
         flagging_dir = self.flagging_dir
         log_filepath = Path(flagging_dir) / "log.csv"
         is_new = not Path(log_filepath).exists()
-        headers = ["query", "image directory", "relevance", "username", "timestamp"]
+        headers = ["query", "selected image", "relevance", "username", "timestamp"]
 
         csv_data = []
         for idx, (component, sample) in enumerate(zip(self.components, flag_data)):
@@ -119,8 +119,9 @@ def search_text(query, top_k=1):
         top_k (int, optional): Amount of images o return]. Defaults to 1.
 
     Returns:
+        [list]: list of images with captions that are related to the query.
         [list]: list of images that are related to the query.
-        [list]: list of image embs that are related to the query.
+        [list]: list of captions with the images that are related to the query.
     """
     logging.info(f"[SearchText]: Searching for {query} with top_k={top_k}...")
     
@@ -133,16 +134,21 @@ def search_text(query, top_k=1):
     # It then returns the top_k highest ranked images, which we output
     hits = util.semantic_search(query_emb, img_emb, top_k=top_k)[0]
 
-    image = []
+    image_caption = []
+    images = []
+    captions = []
     for hit in hits:
         #print(img_names[hit['corpus_id']])
         object = Image.open(os.path.join(
             "photos/", img_names[hit['corpus_id']]))
-        image.append(object)
+        caption = ""
+        image_caption.append((object, caption))
+        images.append(object)
+        captions.append(caption)
         # selected_image_embs.append(img_emb[hit['corpus_id']])
         #print(f'array length is: {len(image)}')
-    logging.info(f"[SearchText]: Found {len(image)} images.")
-    return image
+    logging.info(f"[SearchText]: Found {len(image_caption)} images.")
+    return image_caption, images, captions
 
 # def select_image(evt: gr.SelectData):
 #     """ Returns the index of the selected image
@@ -181,44 +187,52 @@ with gr.Blocks() as demo:
     )
     with gr.Row():
         with gr.Column():
-            query = gr.Textbox(lines=4, 
-                                label="Write what you are looking for in an image...",
-                                placeholder="Text Here...")
-            top_k = gr.Slider(0, 5, step=1, label="Top K relevant images to show")
-            username = gr.Textbox(lines=1, label="Input your unique username 👻 ", placeholder="Text username here...")
+            query = gr.Textbox(lines=4,
+                               label="Query",
+                               placeholder="Text Here...")
+            top_k = gr.Slider(0, 5, step=1, label="Top K")
+            username = gr.Textbox(lines=1, label="Your Name",
+                                  placeholder="Text username here...")
+            submit_btn = gr.Button("Submit")
         with gr.Column():
             gallery = gr.Gallery(
                 label="Generated images", show_label=False, elem_id="gallery"
-                ).style(grid=[3], height="auto")
-            relevance = gr.Dropdown([str(i) for i in range(6)], multiselect=False,
-                                    label="How relevent is this image to your input text?")
-    with gr.Row():
-        with gr.Column():
-            submit_btn = gr.Button("Submit")
-        with gr.Column():
-            save_btn = gr.Button("Save after you select the relevance of each image")
+            ).style(grid=[3], height="auto")
+            t = gr.Textbox(label="Image Caption")
+            relevance = gr.Dropdown(
+                ["0: Not relevant",
+                 "1: Related but not relevant",
+                 "2: Somehow relevant",
+                 "3: Highly relevant"
+                 ], multiselect=False,
+                label="How relevent is this image?"
+            )
+            save_btn = gr.Button(
+                "Save after you select the relevance of each image")
+            
+            i = gr.Image(interactive=False, label="Selected Image", visible=False)
+    
     gr.Markdown("## Here are some examples you can use:")
     gr.Examples(examples, [query, top_k, username])
 
-    callback.setup([query, gallery, relevance, username], "flagged")
-    
+    # states for passing images and texts to other blocks
+    images = gr.State()
+    texts = gr.State()
     # when user input query and top_k
-    submit_btn.click(search_text, [query, top_k], [gallery])
-    
-    # image_relevance_state = gr.State(value={})
-    # selected_index = gr.Number(value=0, visible=False, precision=0)
-    
-    # when user select an image in the gallery
-    # gallery.select(select_image, None, selected_index)
-    # when user select the relevance of the image
-    # relevance.select(fn=select_image_relevance, 
-    #                  inputs=[gallery, selected_index, image_relevance_state], 
-    #                  outputs=image_relevance_state)
-    
+    submit_btn.click(search_text, [query, top_k], [gallery, images, texts])
+
+    # Selected event
+    def display(images, texts, event_data: gr.SelectData):
+        return images[event_data.index], texts[event_data.index]
+
+    selected = gr.State()
+    gallery.select(display, [images, texts], [i, t])
+
     # when user click save button
     # we will flag the current query, selected image, relevance, and username
-    save_btn.click(lambda *args: callback.flag(args), [query, gallery, relevance, username], preprocess=False)
-    # gallery_embs = []
+    callback.setup([query, i, relevance, username], "flagged")
+    save_btn.click(lambda *args: callback.flag(args),
+                   [query, i, relevance, username], preprocess=False)
 
     gr.Markdown(
         """

flagged/Generated images/0b8357f6-acc7-4c33-b72e-dd2eb4a7cdf8/captions.json

@@ -1 +0,0 @@
-{"/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/0b8357f6-acc7-4c33-b72e-dd2eb4a7cdf8/tmpjh4qmiy6mwsrfocc.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/0b8357f6-acc7-4c33-b72e-dd2eb4a7cdf8/tmpksur7bpvwgyoyw1k.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/0b8357f6-acc7-4c33-b72e-dd2eb4a7cdf8/tmpdfeak6vzw2cagssp.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/0b8357f6-acc7-4c33-b72e-dd2eb4a7cdf8/tmpa7vkm9efj35npt_l.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/0b8357f6-acc7-4c33-b72e-dd2eb4a7cdf8/tmptosoqe1qhyk1_go7.png": null}

flagged/Generated images/1ea3115e-3819-4e59-960b-b4ee640f1d05/captions.json

@@ -1 +0,0 @@
-{"/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/1ea3115e-3819-4e59-960b-b4ee640f1d05/tmpjh4qmiy67e8523_b.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/1ea3115e-3819-4e59-960b-b4ee640f1d05/tmpksur7bpvc5fv9_5d.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/1ea3115e-3819-4e59-960b-b4ee640f1d05/tmpdfeak6vztzl2_ws1.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/1ea3115e-3819-4e59-960b-b4ee640f1d05/tmpa7vkm9efphhob5x8.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/1ea3115e-3819-4e59-960b-b4ee640f1d05/tmptosoqe1qa_e8gmq3.png": null}

flagged/Generated images/4f99a65b-30b3-4180-a5d0-c6bd26fbb8ea/captions.json

@@ -1 +0,0 @@
-{"/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/4f99a65b-30b3-4180-a5d0-c6bd26fbb8ea/tmpjofff9lh8ir592_s.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/4f99a65b-30b3-4180-a5d0-c6bd26fbb8ea/tmpq_u5_esu0xqahhd3.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/4f99a65b-30b3-4180-a5d0-c6bd26fbb8ea/tmpvxeix4sb21cqtbci.png": null}

flagged/Generated images/711c188c-154c-4269-8570-ea0be200bbd7/captions.json

@@ -1 +0,0 @@
-{"/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/711c188c-154c-4269-8570-ea0be200bbd7/tmpjh4qmiy6iqdrqkfy.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/711c188c-154c-4269-8570-ea0be200bbd7/tmpksur7bpv4_rm892y.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/711c188c-154c-4269-8570-ea0be200bbd7/tmpdfeak6vzlwes348w.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/711c188c-154c-4269-8570-ea0be200bbd7/tmpa7vkm9efkqe7xs80.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/711c188c-154c-4269-8570-ea0be200bbd7/tmptosoqe1qy9zed0qk.png": null}

flagged/Generated images/aeb09d59-e4b7-4cb0-b36e-c0af8ee754fc/captions.json

@@ -1 +0,0 @@
-{"/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/aeb09d59-e4b7-4cb0-b36e-c0af8ee754fc/tmptv98_joyv5gkcmxq.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/aeb09d59-e4b7-4cb0-b36e-c0af8ee754fc/tmpwcii60a869asmn2a.png": null}

flagged/Generated images/b9317495-4a7a-4290-90e2-99e6b9f3cbcf/captions.json

@@ -1 +0,0 @@
-{"/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/b9317495-4a7a-4290-90e2-99e6b9f3cbcf/tmptv98_joy46qg2bqy.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/b9317495-4a7a-4290-90e2-99e6b9f3cbcf/tmpwcii60a8aon_3pr0.png": null}

flagged/Generated images/d3e86f79-b53a-44e0-b982-88c4d63a46d4/captions.json

@@ -1 +0,0 @@
-{"/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/d3e86f79-b53a-44e0-b982-88c4d63a46d4/tmpjh4qmiy6kgu2tl2e.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/d3e86f79-b53a-44e0-b982-88c4d63a46d4/tmpksur7bpvj67d3h1w.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/d3e86f79-b53a-44e0-b982-88c4d63a46d4/tmpdfeak6vzk41q9b3l.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/d3e86f79-b53a-44e0-b982-88c4d63a46d4/tmpa7vkm9ef7vniucwu.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/d3e86f79-b53a-44e0-b982-88c4d63a46d4/tmptosoqe1qwlz19a2i.png": null}

flagged/Generated images/fd46c482-8413-411e-b4ac-f573484cb31d/captions.json

@@ -1 +0,0 @@
-{"/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/fd46c482-8413-411e-b4ac-f573484cb31d/tmpjh4qmiy60gq4dw4r.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/fd46c482-8413-411e-b4ac-f573484cb31d/tmpksur7bpvp2uhgv2g.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/fd46c482-8413-411e-b4ac-f573484cb31d/tmpdfeak6vzxvz6t_vu.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/fd46c482-8413-411e-b4ac-f573484cb31d/tmpa7vkm9efq_k8xsys.png": null, "/Users/zhilinzhou/Local-documents/Workspace/CLIP-image-search/flagged/Generated images/fd46c482-8413-411e-b4ac-f573484cb31d/tmptosoqe1q1fskh25b.png": null}

flagged/log.csv

@@ -1,9 +1,6 @@
-query,image directory,relevance,username,timestamp
-A cute kangaroo,711c188c-154c-4269-8570-ea0be200bbd7,5,ghost,2023-04-05 22:48:25.682888
-A cute kangaroo,fd46c482-8413-411e-b4ac-f573484cb31d,2,ghost,2023-04-05 22:48:30.769311
-A cute kangaroo,0b8357f6-acc7-4c33-b72e-dd2eb4a7cdf8,2,ghost,2023-04-05 22:48:32.753789
-A cute kangaroo,d3e86f79-b53a-44e0-b982-88c4d63a46d4,3,ghost,2023-04-05 22:48:35.323199
-A cute kangaroo,1ea3115e-3819-4e59-960b-b4ee640f1d05,5,ghost,2023-04-05 22:48:37.470948
-Dog in the beach,aeb09d59-e4b7-4cb0-b36e-c0af8ee754fc,5,ghost,2023-04-05 22:50:38.332683
-Dog in the beach,b9317495-4a7a-4290-90e2-99e6b9f3cbcf,2,ghost,2023-04-05 22:51:24.389490
-un homme marchant sur le parc,4f99a65b-30b3-4180-a5d0-c6bd26fbb8ea,3,ghost,2023-04-05 22:59:30.956854
+query,selected image,relevance,username,timestamp
+A cute kangaroo,tmpzsbu_wpf.png,3: Highly relevant,ghost,2023-04-11 09:34:45.422185
+A cute kangaroo,tmpor4tmvfk.png,1: Related but not relevant,ghost,2023-04-11 09:34:51.574117
+A cute kangaroo,tmpt52dvzm3.png,1: Related but not relevant,ghost,2023-04-11 09:34:55.734014
+A cute kangaroo,tmpbs578cha.png,2: Somehow relevant,ghost,2023-04-11 09:35:02.914171
+A cute kangaroo,tmpsyidqwtr.png,2: Somehow relevant,ghost,2023-04-11 09:35:05.194639