revert

app.py

@@ -2,20 +2,18 @@ import os
 
 import gradio as gr
 import torch
-from colpali_engine.models.paligemma_colbert_architecture import ColPali
-from colpali_engine.trainer.retrieval_evaluator import CustomEvaluator
-from colpali_engine.utils.colpali_processing_utils import (
-    process_images,
-    process_queries,
-)
 from pdf2image import convert_from_path
 from PIL import Image
 from torch.utils.data import DataLoader
 from tqdm import tqdm
 from transformers import AutoProcessor
 
+from colpali_engine.models.paligemma_colbert_architecture import ColPali
+from colpali_engine.trainer.retrieval_evaluator import CustomEvaluator
+from colpali_engine.utils.colpali_processing_utils import process_images, process_queries
+
 
-def search(query: str, ds, images, k):
+def search(query: str, ds, images):
     qs = []
     with torch.no_grad():
         batch_query = process_queries(processor, [query], mock_image)
@@ -23,27 +21,19 @@ def search(query: str, ds, images, k):
         embeddings_query = model(**batch_query)
         qs.extend(list(torch.unbind(embeddings_query.to("cpu"))))
 
+    # run evaluation
     retriever_evaluator = CustomEvaluator(is_multi_vector=True)
     scores = retriever_evaluator.evaluate(qs, ds)
+    best_page = int(scores.argmax(axis=1).item())
+    return f"The most relevant page is {best_page}", images[best_page]
 
-    top_k_indices = scores.argsort(axis=1)[0][-k:][::-1]
 
-    results = []
-    for idx in top_k_indices:
-        results.append((images[idx], f"Page {idx}"))
-
-    return results
-
-
-def index(files, ds):
+def index(file, ds):
     """Example script to run inference with ColPali"""
     images = []
-    for f in files:
+    for f in file:
         images.extend(convert_from_path(f))
 
-    if len(images) >= 150:
-        raise gr.Error("The number of images in the dataset should be less than 150.")
-
     # run inference - docs
     dataloader = DataLoader(
         images,
@@ -58,50 +48,41 @@ def index(files, ds):
         ds.extend(list(torch.unbind(embeddings_doc.to("cpu"))))
     return f"Uploaded and converted {len(images)} pages", ds, images
 
-cache_dir = os.path.join(os.getcwd(), "data/", "model_cache/")
+
+COLORS = ["#4285f4", "#db4437", "#f4b400", "#0f9d58", "#e48ef1"]
 # Load model
 model_name = "vidore/colpali"
 token = os.environ.get("HF_TOKEN")
 model = ColPali.from_pretrained(
-    "google/paligemma-3b-mix-448", torch_dtype=torch.bfloat16, device_map="cuda", token = token, cache_dir=cache_dir
+    "google/paligemma-3b-mix-448", torch_dtype=torch.bfloat16, device_map="cuda", token=token
 ).eval()
-
 model.load_adapter(model_name)
-processor = AutoProcessor.from_pretrained(model_name, cache_dir=cache_dir, token = token)
-
+processor = AutoProcessor.from_pretrained(model_name, token=token)
 device = model.device
-
 mock_image = Image.new("RGB", (448, 448), (255, 255, 255))
 
-with gr.Blocks(theme=gr.themes.Soft()) as demo:
-    gr.Markdown("# ColPali: Efficient Document Retrieval with Vision Language Models 📚")
-    gr.Markdown("""Demo to test ColPali on PDF documents. The inference code is based on the [ViDoRe benchmark](https://github.com/illuin-tech/vidore-benchmark).
-
-    ColPali is model implemented from the [ColPali paper](https://arxiv.org/abs/2407.01449).
+with gr.Blocks() as demo:
+    gr.Markdown("# ColPali: Efficient Document Retrieval with Vision Language Models 📚🔍")
+    gr.Markdown("## 1️⃣ Upload PDFs")
+    file = gr.File(file_types=["pdf"], file_count="multiple")
 
-    This demo allows you to upload PDF files and search for the most relevant pages based on your query.
-    """)
-    with gr.Row():
-        with gr.Column(scale=2):
-            gr.Markdown("## 1️⃣ Upload PDFs")
-            file = gr.File(file_types=["pdf"], file_count="multiple", label="Upload PDFs")
+    gr.Markdown("## 2️⃣ Convert the PDFs and upload")
+    convert_button = gr.Button("🔄 Convert and upload")
+    message = gr.Textbox("Files not yet uploaded")
+    embeds = gr.State(value=[])
+    imgs = gr.State(value=[])
 
-            convert_button = gr.Button("🔄 Convert and upload")
-            message = gr.Textbox("Files not yet uploaded", label="Status")
-            embeds = gr.State(value=[])
-            imgs = gr.State(value=[])
+    # Define the actions
+    convert_button.click(index, inputs=[file, embeds], outputs=[message, embeds, imgs])
 
-        with gr.Column(scale=3):
-            gr.Markdown("## 2️⃣ Search")
-            query = gr.Textbox(placeholder="Enter your query here", label="Query")
-            k = gr.Slider(minimum=1, maximum=10, step=1, label="Number of results", value=3)
+    gr.Markdown("## 3️⃣ Search")
+    query = gr.Textbox(placeholder="Enter your query here")
+    search_button = gr.Button("🔍 Search")
+    message2 = gr.Textbox("Query not yet set")
+    output_img = gr.Image()
 
-    # Define the actions
-    search_button = gr.Button("🔍 Search", variant="primary")
-    output_gallery = gr.Gallery(label="Retrieved Documents", height=600, show_label=True)
+    search_button.click(search, inputs=[query, embeds, imgs], outputs=[message2, output_img])
 
-    convert_button.click(index, inputs=[file, embeds], outputs=[message, embeds, imgs])
-    search_button.click(search, inputs=[query, embeds, imgs, k], outputs=[output_gallery])
 
 if __name__ == "__main__":
-    demo.queue(max_size=10).launch(debug=True, server_name="0.0.0.0", server_port=7861)
+    demo.queue(max_size=10).launch(debug=True)