app.py

import os
import random
import gradio as gr
import torch
import clip
import numpy as np
import pandas as pd



device = "mps" if torch.backends.mps.is_available() else "cpu"
model, preprocess = clip.load("ViT-B/32", device=device)
print('Using ' + device)

features_path = 'features/'



photo_features = np.load(features_path + "features.npy")
photo_ids = pd.read_csv(features_path+ "updated_file.csv")
descriptions = list(photo_ids['description'])
photo_filenames = list(photo_ids['photo_id'])



def clip_search(search_string):
        
    with torch.no_grad():
        # Encode and normalize the description using CLIP
        text_encoded = model.encode_text(clip.tokenize(search_string).to(device))
        text_encoded /= text_encoded.norm(dim=-1, keepdim=True)
        # Retrieve the description vector and the photo vectors
    text_features = text_encoded.cpu().numpy()

    # Compute the similarity between the descrption and each photo using the Cosine similarity
    similarities = list((text_features @ photo_features.T).squeeze(0))

    # Sort the photos by their similarity score
    candidates = sorted(zip(similarities, range(photo_features.shape[0])), key=lambda x: x[0], reverse=True)
    
    images = []
    for i in range(30):
        # Retrieve the photo ID
        idx = candidates[i][1]
        photo_id = photo_filenames[idx]
        caption = descriptions[idx]

        images.append([('https://thegogglesdonothing.com/projects/clipsearch/StanfordVRC/images/' + str(photo_id)),  caption])
    return images

css = "footer {display: none !important;} .gradio-container {min-height: 0px !important;}"
with gr.Blocks(css = css) as demo:
    with gr.Column(variant="panel"):
        with gr.Row(variant="compact"):
            search_string = gr.Textbox(
                label="Evocative Search",
                show_label=True,
                max_lines=1,
                placeholder="Type something abstruse, or click a suggested search below.",
            )
            btn = gr.Button("Retrieve Images", variant="primary")
        with gr.Row(variant="compact"):
            suggest1 = gr.Button("rococo", variant="secondary")    
            suggest2 = gr.Button("brutalism", variant="secondary")    
            suggest3 = gr.Button("classical", variant="secondary")       
            suggest4 = gr.Button("gothic", variant="secondary")    
            suggest5 = gr.Button("foliate", variant="secondary")  
        gallery = gr.Gallery(
            label=False, show_label=False, elem_id="gallery", columns=[6]
        )

    suggest1.click(clip_search, inputs=suggest1, outputs=gallery)
    suggest2.click(clip_search, inputs=suggest2, outputs=gallery)
    suggest3.click(clip_search, inputs=suggest3, outputs=gallery)
    suggest4.click(clip_search, inputs=suggest4, outputs=gallery)
    suggest5.click(clip_search, inputs=suggest5, outputs=gallery)
    btn.click(clip_search, inputs=search_string, outputs=gallery)
    search_string.submit(clip_search, search_string, gallery)



if __name__ == "__main__":
     demo.launch()