Spaces:
Sleeping
Sleeping
| import os | |
| import random | |
| import gradio as gr | |
| import torch | |
| import clip | |
| import numpy as np | |
| import pandas as pd | |
| device = "mps" if torch.backends.mps.is_available() else "cpu" | |
| model, preprocess = clip.load("ViT-B/32", device=device) | |
| print('Using ' + device) | |
| features_path = 'features/' | |
| photo_features = np.load(features_path + "features.npy") | |
| photo_ids = pd.read_csv(features_path+ "updated_file.csv") | |
| descriptions = list(photo_ids['description']) | |
| photo_filenames = list(photo_ids['photo_id']) | |
| def clip_search(search_string): | |
| with torch.no_grad(): | |
| # Encode and normalize the description using CLIP | |
| text_encoded = model.encode_text(clip.tokenize(search_string).to(device)) | |
| text_encoded /= text_encoded.norm(dim=-1, keepdim=True) | |
| # Retrieve the description vector and the photo vectors | |
| text_features = text_encoded.cpu().numpy() | |
| # Compute the similarity between the descrption and each photo using the Cosine similarity | |
| similarities = list((text_features @ photo_features.T).squeeze(0)) | |
| # Sort the photos by their similarity score | |
| candidates = sorted(zip(similarities, range(photo_features.shape[0])), key=lambda x: x[0], reverse=True) | |
| images = [] | |
| for i in range(30): | |
| # Retrieve the photo ID | |
| idx = candidates[i][1] | |
| photo_id = photo_filenames[idx] | |
| caption = descriptions[idx] | |
| images.append([('https://thegogglesdonothing.com/projects/clipsearch/StanfordVRC/images/' + str(photo_id)), caption]) | |
| return images | |
| css = "footer {display: none !important;} .gradio-container {min-height: 0px !important;}" | |
| with gr.Blocks(css = css) as demo: | |
| with gr.Column(variant="panel"): | |
| with gr.Row(variant="compact"): | |
| search_string = gr.Textbox( | |
| label="Evocative Search", | |
| show_label=True, | |
| max_lines=1, | |
| placeholder="Type something abstruse, or click a suggested search below.", | |
| ) | |
| btn = gr.Button("Retrieve Images", variant="primary") | |
| with gr.Row(variant="compact"): | |
| suggest1 = gr.Button("rococo", variant="secondary") | |
| suggest2 = gr.Button("brutalism", variant="secondary") | |
| suggest3 = gr.Button("classical", variant="secondary") | |
| suggest4 = gr.Button("gothic", variant="secondary") | |
| suggest5 = gr.Button("foliate", variant="secondary") | |
| gallery = gr.Gallery( | |
| label=False, show_label=False, elem_id="gallery", columns=[6] | |
| ) | |
| suggest1.click(clip_search, inputs=suggest1, outputs=gallery) | |
| suggest2.click(clip_search, inputs=suggest2, outputs=gallery) | |
| suggest3.click(clip_search, inputs=suggest3, outputs=gallery) | |
| suggest4.click(clip_search, inputs=suggest4, outputs=gallery) | |
| suggest5.click(clip_search, inputs=suggest5, outputs=gallery) | |
| btn.click(clip_search, inputs=search_string, outputs=gallery) | |
| search_string.submit(clip_search, search_string, gallery) | |
| if __name__ == "__main__": | |
| demo.launch() | |