feat : add retrieve top-k + improve app style (#1)

- feat : add retrieve top-k + improve app style (fc4a494b79a42515269d663d85dea666184d9123)


Co-authored-by: Hugues Sibille <[email protected]>

app.py

@@ -2,18 +2,20 @@ import os
 
 import gradio as gr
 import torch
+from colpali_engine.models.paligemma_colbert_architecture import ColPali
+from colpali_engine.trainer.retrieval_evaluator import CustomEvaluator
+from colpali_engine.utils.colpali_processing_utils import (
+    process_images,
+    process_queries,
+)
 from pdf2image import convert_from_path
 from PIL import Image
 from torch.utils.data import DataLoader
 from tqdm import tqdm
 from transformers import AutoProcessor
 
-from colpali_engine.models.paligemma_colbert_architecture import ColPali
-from colpali_engine.trainer.retrieval_evaluator import CustomEvaluator
-from colpali_engine.utils.colpali_processing_utils import process_images, process_queries
-
 
-def search(query: str, ds, images):
+def search(query: str, ds, images, k):
     qs = []
     with torch.no_grad():
         batch_query = process_queries(processor, [query], mock_image)
@@ -21,19 +23,27 @@ def search(query: str, ds, images):
         embeddings_query = model(**batch_query)
         qs.extend(list(torch.unbind(embeddings_query.to("cpu"))))
 
-    # run evaluation
     retriever_evaluator = CustomEvaluator(is_multi_vector=True)
     scores = retriever_evaluator.evaluate(qs, ds)
-    best_page = int(scores.argmax(axis=1).item())
-    return f"The most relevant page is {best_page}", images[best_page]
 
+    top_k_indices = scores.argsort(axis=1)[0][-k:][::-1]
 
-def index(file, ds):
+    results = []
+    for idx in top_k_indices:
+        results.append((images[idx], f"Page {idx}"))
+
+    return results
+
+
+def index(files, ds):
     """Example script to run inference with ColPali"""
     images = []
-    for f in file:
+    for f in files:
         images.extend(convert_from_path(f))
 
+    if len(images) >= 150:
+        raise gr.Error("The number of images in the dataset should be less than 150.")
+
     # run inference - docs
     dataloader = DataLoader(
         images,
@@ -48,41 +58,50 @@ def index(file, ds):
         ds.extend(list(torch.unbind(embeddings_doc.to("cpu"))))
     return f"Uploaded and converted {len(images)} pages", ds, images
 
-
-COLORS = ["#4285f4", "#db4437", "#f4b400", "#0f9d58", "#e48ef1"]
+cache_dir = os.path.join(os.getcwd(), "data/", "model_cache/")
 # Load model
 model_name = "vidore/colpali"
 token = os.environ.get("HF_TOKEN")
 model = ColPali.from_pretrained(
-    "google/paligemma-3b-mix-448", torch_dtype=torch.bfloat16, device_map="cuda", token=token
+    "google/paligemma-3b-mix-448", torch_dtype=torch.bfloat16, device_map="cuda", token = token, cache_dir=cache_dir
 ).eval()
+
 model.load_adapter(model_name)
-processor = AutoProcessor.from_pretrained(model_name, token=token)
+processor = AutoProcessor.from_pretrained(model_name, cache_dir=cache_dir, token = token)
+
 device = model.device
+
 mock_image = Image.new("RGB", (448, 448), (255, 255, 255))
 
-with gr.Blocks() as demo:
-    gr.Markdown("# ColPali: Efficient Document Retrieval with Vision Language Models 📚🔍")
-    gr.Markdown("## 1️⃣ Upload PDFs")
-    file = gr.File(file_types=["pdf"], file_count="multiple")
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    gr.Markdown("# ColPali: Efficient Document Retrieval with Vision Language Models 📚")
+    gr.Markdown("""Demo to test ColPali on PDF documents. The inference code is based on the [ViDoRe benchmark](https://github.com/illuin-tech/vidore-benchmark).
 
-    gr.Markdown("## 2️⃣ Convert the PDFs and upload")
-    convert_button = gr.Button("🔄 Convert and upload")
-    message = gr.Textbox("Files not yet uploaded")
-    embeds = gr.State(value=[])
-    imgs = gr.State(value=[])
+    ColPali is model implemented from the [ColPali paper](https://arxiv.org/abs/2407.01449).
 
-    # Define the actions
-    convert_button.click(index, inputs=[file, embeds], outputs=[message, embeds, imgs])
+    This demo allows you to upload PDF files and search for the most relevant pages based on your query.
+    """)
+    with gr.Row():
+        with gr.Column(scale=2):
+            gr.Markdown("## 1️⃣ Upload PDFs")
+            file = gr.File(file_types=["pdf"], file_count="multiple", label="Upload PDFs")
+
+            convert_button = gr.Button("🔄 Convert and upload")
+            message = gr.Textbox("Files not yet uploaded", label="Status")
+            embeds = gr.State(value=[])
+            imgs = gr.State(value=[])
 
-    gr.Markdown("## 3️⃣ Search")
-    query = gr.Textbox(placeholder="Enter your query here")
-    search_button = gr.Button("🔍 Search")
-    message2 = gr.Textbox("Query not yet set")
-    output_img = gr.Image()
+        with gr.Column(scale=3):
+            gr.Markdown("## 2️⃣ Search")
+            query = gr.Textbox(placeholder="Enter your query here", label="Query")
+            k = gr.Slider(minimum=1, maximum=10, step=1, label="Number of results", value=3)
 
-    search_button.click(search, inputs=[query, embeds, imgs], outputs=[message2, output_img])
+    # Define the actions
+    search_button = gr.Button("🔍 Search", variant="primary")
+    output_gallery = gr.Gallery(label="Retrieved Documents", height=600, show_label=True)
 
+    convert_button.click(index, inputs=[file, embeds], outputs=[message, embeds, imgs])
+    search_button.click(search, inputs=[query, embeds, imgs, k], outputs=[output_gallery])
 
 if __name__ == "__main__":
-    demo.queue(max_size=10).launch(debug=True)
+    demo.queue(max_size=10).launch(debug=True, server_name="0.0.0.0", server_port=7861)