Spaces:
Running
Running
| <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head> | |
| <meta charset="utf-8"> | |
| <meta name="generator" content="quarto-1.6.40"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes"> | |
| <title>Similarity Search – Open-Source AI Cookbook</title> | |
| <style> | |
| code{white-space: pre-wrap;} | |
| span.smallcaps{font-variant: small-caps;} | |
| div.columns{display: flex; gap: min(4vw, 1.5em);} | |
| div.column{flex: auto; overflow-x: auto;} | |
| div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;} | |
| ul.task-list{list-style: none;} | |
| ul.task-list li input[type="checkbox"] { | |
| width: 0.8em; | |
| margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */ | |
| vertical-align: middle; | |
| } | |
| /* CSS for syntax highlighting */ | |
| pre > code.sourceCode { white-space: pre; position: relative; } | |
| pre > code.sourceCode > span { line-height: 1.25; } | |
| pre > code.sourceCode > span:empty { height: 1.2em; } | |
| .sourceCode { overflow: visible; } | |
| code.sourceCode > span { color: inherit; text-decoration: inherit; } | |
| div.sourceCode { margin: 1em 0; } | |
| pre.sourceCode { margin: 0; } | |
| @media screen { | |
| div.sourceCode { overflow: auto; } | |
| } | |
| @media print { | |
| pre > code.sourceCode { white-space: pre-wrap; } | |
| pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } | |
| } | |
| pre.numberSource code | |
| { counter-reset: source-line 0; } | |
| pre.numberSource code > span | |
| { position: relative; left: -4em; counter-increment: source-line; } | |
| pre.numberSource code > span > a:first-child::before | |
| { content: counter(source-line); | |
| position: relative; left: -1em; text-align: right; vertical-align: baseline; | |
| border: none; display: inline-block; | |
| -webkit-touch-callout: none; -webkit-user-select: none; | |
| -khtml-user-select: none; -moz-user-select: none; | |
| -ms-user-select: none; user-select: none; | |
| padding: 0 4px; width: 4em; | |
| } | |
| pre.numberSource { margin-left: 3em; padding-left: 4px; } | |
| div.sourceCode | |
| { } | |
| @media screen { | |
| pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; } | |
| } | |
| </style> | |
| <script src="../site_libs/quarto-nav/quarto-nav.js"></script> | |
| <script src="../site_libs/quarto-nav/headroom.min.js"></script> | |
| <script src="../site_libs/clipboard/clipboard.min.js"></script> | |
| <script src="../site_libs/quarto-search/autocomplete.umd.js"></script> | |
| <script src="../site_libs/quarto-search/fuse.min.js"></script> | |
| <script src="../site_libs/quarto-search/quarto-search.js"></script> | |
| <meta name="quarto:offset" content="../"> | |
| <script src="../site_libs/quarto-html/quarto.js"></script> | |
| <script src="../site_libs/quarto-html/popper.min.js"></script> | |
| <script src="../site_libs/quarto-html/tippy.umd.min.js"></script> | |
| <script src="../site_libs/quarto-html/anchor.min.js"></script> | |
| <link href="../site_libs/quarto-html/tippy.css" rel="stylesheet"> | |
| <link href="../site_libs/quarto-html/quarto-syntax-highlighting-549806ee2085284f45b00abea8c6df48.css" rel="stylesheet" id="quarto-text-highlighting-styles"> | |
| <script src="../site_libs/bootstrap/bootstrap.min.js"></script> | |
| <link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet"> | |
| <link href="../site_libs/bootstrap/bootstrap-2be10d9e998f81ff6e49e26833438aa5.min.css" rel="stylesheet" append-hash="true" id="quarto-bootstrap" data-mode="light"> | |
| <script id="quarto-search-options" type="application/json">{ | |
| "location": "sidebar", | |
| "copy-button": false, | |
| "collapse-after": 3, | |
| "panel-placement": "start", | |
| "type": "textbox", | |
| "limit": 50, | |
| "keyboard-shortcut": [ | |
| "f", | |
| "/", | |
| "s" | |
| ], | |
| "show-item-context": false, | |
| "language": { | |
| "search-no-results-text": "No results", | |
| "search-matching-documents-text": "matching documents", | |
| "search-copy-link-title": "Copy link to search", | |
| "search-hide-matches-text": "Hide additional matches", | |
| "search-more-match-text": "more match in this document", | |
| "search-more-matches-text": "more matches in this document", | |
| "search-clear-button-title": "Clear", | |
| "search-text-placeholder": "", | |
| "search-detached-cancel-button-title": "Cancel", | |
| "search-submit-button-title": "Submit", | |
| "search-label": "Search" | |
| } | |
| }</script> | |
| <link rel="stylesheet" href="../styles.css"> | |
| </head> | |
| <body class="nav-sidebar docked"> | |
| <div id="quarto-search-results"></div> | |
| <header id="quarto-header" class="headroom fixed-top"> | |
| <nav class="quarto-secondary-nav"> | |
| <div class="container-fluid d-flex"> | |
| <button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" role="button" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }"> | |
| <i class="bi bi-layout-text-sidebar-reverse"></i> | |
| </button> | |
| <nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item">Open-Source AI Cookbook</li><li class="breadcrumb-item"><a href="../notebooks/automatic_embedding.html">Additional Techniques</a></li><li class="breadcrumb-item"><a href="../notebooks/faiss.html">FAISS for Efficient Search</a></li></ol></nav> | |
| <a class="flex-grow-1" role="navigation" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }"> | |
| </a> | |
| <button type="button" class="btn quarto-search-button" aria-label="Search" onclick="window.quartoOpenSearch();"> | |
| <i class="bi bi-search"></i> | |
| </button> | |
| </div> | |
| </nav> | |
| </header> | |
| <!-- content --> | |
| <div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article"> | |
| <!-- sidebar --> | |
| <nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation docked overflow-auto"> | |
| <div class="pt-lg-2 mt-2 text-left sidebar-header"> | |
| <div class="sidebar-title mb-0 py-0"> | |
| <a href="../">Open-Source AI Cookbook</a> | |
| </div> | |
| </div> | |
| <div class="mt-2 flex-shrink-0 align-items-center"> | |
| <div class="sidebar-search"> | |
| <div id="quarto-search" class="" title="Search"></div> | |
| </div> | |
| </div> | |
| <div class="sidebar-menu-container"> | |
| <ul class="list-unstyled mt-1"> | |
| <li class="sidebar-item sidebar-item-section"> | |
| <div class="sidebar-item-container"> | |
| <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true"> | |
| <span class="menu-text">About</span></a> | |
| <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true" aria-label="Toggle section"> | |
| <i class="bi bi-chevron-right ms-2"></i> | |
| </a> | |
| </div> | |
| <ul id="quarto-sidebar-section-1" class="collapse list-unstyled sidebar-section depth1 show"> | |
| <li class="sidebar-item"> | |
| <div class="sidebar-item-container"> | |
| <a href="../index.html" class="sidebar-item-text sidebar-link"> | |
| <span class="menu-text">About Quarto</span></a> | |
| </div> | |
| </li> | |
| </ul> | |
| </li> | |
| <li class="sidebar-item sidebar-item-section"> | |
| <div class="sidebar-item-container"> | |
| <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true"> | |
| <span class="menu-text">Open-Source AI Cookbook</span></a> | |
| <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true" aria-label="Toggle section"> | |
| <i class="bi bi-chevron-right ms-2"></i> | |
| </a> | |
| </div> | |
| <ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show"> | |
| <li class="sidebar-item sidebar-item-section"> | |
| <div class="sidebar-item-container"> | |
| <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-3" role="navigation" aria-expanded="true"> | |
| <span class="menu-text">RAG Techniques</span></a> | |
| <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-3" role="navigation" aria-expanded="true" aria-label="Toggle section"> | |
| <i class="bi bi-chevron-right ms-2"></i> | |
| </a> | |
| </div> | |
| <ul id="quarto-sidebar-section-3" class="collapse list-unstyled sidebar-section depth2 show"> | |
| <li class="sidebar-item"> | |
| <div class="sidebar-item-container"> | |
| <a href="../notebooks/rag_zephyr_langchain.html" class="sidebar-item-text sidebar-link"> | |
| <span class="menu-text">RAG Zephyr & LangChain</span></a> | |
| </div> | |
| </li> | |
| <li class="sidebar-item"> | |
| <div class="sidebar-item-container"> | |
| <a href="../notebooks/advanced_rag.html" class="sidebar-item-text sidebar-link"> | |
| <span class="menu-text">Advanced RAG</span></a> | |
| </div> | |
| </li> | |
| <li class="sidebar-item"> | |
| <div class="sidebar-item-container"> | |
| <a href="../notebooks/rag_evaluation.html" class="sidebar-item-text sidebar-link"> | |
| <span class="menu-text">RAG Evaluation</span></a> | |
| </div> | |
| </li> | |
| </ul> | |
| </li> | |
| <li class="sidebar-item sidebar-item-section"> | |
| <div class="sidebar-item-container"> | |
| <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-4" role="navigation" aria-expanded="true"> | |
| <span class="menu-text">Additional Techniques</span></a> | |
| <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-4" role="navigation" aria-expanded="true" aria-label="Toggle section"> | |
| <i class="bi bi-chevron-right ms-2"></i> | |
| </a> | |
| </div> | |
| <ul id="quarto-sidebar-section-4" class="collapse list-unstyled sidebar-section depth2 show"> | |
| <li class="sidebar-item"> | |
| <div class="sidebar-item-container"> | |
| <a href="../notebooks/automatic_embedding.html" class="sidebar-item-text sidebar-link"> | |
| <span class="menu-text">Automatic Embedding</span></a> | |
| </div> | |
| </li> | |
| <li class="sidebar-item"> | |
| <div class="sidebar-item-container"> | |
| <a href="../notebooks/faiss.html" class="sidebar-item-text sidebar-link active"> | |
| <span class="menu-text">FAISS for Efficient Search</span></a> | |
| </div> | |
| </li> | |
| <li class="sidebar-item"> | |
| <div class="sidebar-item-container"> | |
| <a href="../notebooks/single_gpu.html" class="sidebar-item-text sidebar-link"> | |
| <span class="menu-text">Single GPU Optimization</span></a> | |
| </div> | |
| </li> | |
| </ul> | |
| </li> | |
| </ul> | |
| </li> | |
| </ul> | |
| </div> | |
| </nav> | |
| <div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div> | |
| <!-- margin-sidebar --> | |
| <div id="quarto-margin-sidebar" class="sidebar margin-sidebar"> | |
| <nav id="TOC" role="doc-toc" class="toc-active"> | |
| <h2 id="toc-title">On this page</h2> | |
| <ul> | |
| <li><a href="#embedding-multimodal-data-for-similarity-search-using-transformers-datasets-and-faiss" id="toc-embedding-multimodal-data-for-similarity-search-using-transformers-datasets-and-faiss" class="nav-link active" data-scroll-target="#embedding-multimodal-data-for-similarity-search-using-transformers-datasets-and-faiss">Embedding multimodal data for similarity search using 🤗 transformers, 🤗 datasets and FAISS</a> | |
| <ul class="collapse"> | |
| <li><a href="#querying-the-data-with-text-prompts" id="toc-querying-the-data-with-text-prompts" class="nav-link" data-scroll-target="#querying-the-data-with-text-prompts">Querying the data with text prompts</a></li> | |
| <li><a href="#querying-the-data-with-image-prompts" id="toc-querying-the-data-with-image-prompts" class="nav-link" data-scroll-target="#querying-the-data-with-image-prompts">Querying the data with image prompts</a></li> | |
| <li><a href="#saving-pushing-and-loading-the-embeddings" id="toc-saving-pushing-and-loading-the-embeddings" class="nav-link" data-scroll-target="#saving-pushing-and-loading-the-embeddings">Saving, pushing and loading the embeddings</a></li> | |
| </ul></li> | |
| </ul> | |
| </nav> | |
| </div> | |
| <!-- main --> | |
| <main class="content" id="quarto-document-content"> | |
| <header id="title-block-header" class="quarto-title-block default"><nav class="quarto-page-breadcrumbs quarto-title-breadcrumbs d-none d-lg-block" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item">Open-Source AI Cookbook</li><li class="breadcrumb-item"><a href="../notebooks/automatic_embedding.html">Additional Techniques</a></li><li class="breadcrumb-item"><a href="../notebooks/faiss.html">FAISS for Efficient Search</a></li></ol></nav> | |
| <div class="quarto-title"> | |
| <h1 class="title">Similarity Search</h1> | |
| </div> | |
| <div class="quarto-title-meta"> | |
| </div> | |
| </header> | |
| <section id="embedding-multimodal-data-for-similarity-search-using-transformers-datasets-and-faiss" class="level1"> | |
| <h1>Embedding multimodal data for similarity search using 🤗 transformers, 🤗 datasets and FAISS</h1> | |
| <p><em>Authored by: <a href="https://huggingface.co/merve">Merve Noyan</a></em></p> | |
| <p>Embeddings are semantically meaningful compressions of information. They can be used to do similarity search, zero-shot classification or simply train a new model. Use cases for similarity search include searching for similar products in e-commerce, content search in social media and more. This notebook walks you through using 🤗transformers, 🤗datasets and FAISS to create and index embeddings from a feature extraction model to later use them for similarity search. Let’s install necessary libraries.</p> | |
| <div id="cell-1" class="cell"> | |
| <div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="op">!</span>pip install <span class="op">-</span>q datasets faiss<span class="op">-</span>gpu transformers sentencepiece</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <p>For this tutorial, we will use <a href="https://huggingface.co/openai/clip-vit-base-patch16">CLIP model</a> to extract the features. CLIP is a revolutionary model that introduced joint training of a text encoder and an image encoder to connect two modalities.</p> | |
| <div id="cell-3" class="cell"> | |
| <div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> torch</span> | |
| <span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> PIL <span class="im">import</span> Image</span> | |
| <span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> transformers <span class="im">import</span> AutoImageProcessor, AutoModel, AutoTokenizer</span> | |
| <span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> faiss</span> | |
| <span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span> | |
| <span id="cb2-6"><a href="#cb2-6" aria-hidden="true" tabindex="-1"></a></span> | |
| <span id="cb2-7"><a href="#cb2-7" aria-hidden="true" tabindex="-1"></a>device <span class="op">=</span> torch.device(<span class="st">'cuda'</span> <span class="cf">if</span> torch.cuda.is_available() <span class="cf">else</span> <span class="st">"cpu"</span>)</span> | |
| <span id="cb2-8"><a href="#cb2-8" aria-hidden="true" tabindex="-1"></a></span> | |
| <span id="cb2-9"><a href="#cb2-9" aria-hidden="true" tabindex="-1"></a>model <span class="op">=</span> AutoModel.from_pretrained(<span class="st">"openai/clip-vit-base-patch16"</span>).to(device)</span> | |
| <span id="cb2-10"><a href="#cb2-10" aria-hidden="true" tabindex="-1"></a>processor <span class="op">=</span> AutoImageProcessor.from_pretrained(<span class="st">"openai/clip-vit-base-patch16"</span>)</span> | |
| <span id="cb2-11"><a href="#cb2-11" aria-hidden="true" tabindex="-1"></a>tokenizer <span class="op">=</span> AutoTokenizer.from_pretrained(<span class="st">"openai/clip-vit-base-patch16"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <p>Load the dataset. To keep this notebook light, we will use a small captioning dataset, <a href="https://huggingface.co/datasets/jmhessel/newyorker_caption_contest">jmhessel/newyorker_caption_contest</a>.</p> | |
| <div id="cell-5" class="cell"> | |
| <div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> datasets <span class="im">import</span> load_dataset</span> | |
| <span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a></span> | |
| <span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a>ds <span class="op">=</span> load_dataset(<span class="st">"jmhessel/newyorker_caption_contest"</span>, <span class="st">"explanation"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <p>See an example.</p> | |
| <div id="cell-7" class="cell" data-outputid="682033f9-da37-4cae-e1bc-4a5fbbb7f2fa"> | |
| <div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>ds[<span class="st">"train"</span>][<span class="dv">0</span>][<span class="st">"image"</span>]</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| <div class="cell-output cell-output-display" data-execution_count="4"> | |
| <div> | |
| <figure class="figure"> | |
| <p><img src="faiss_files/figure-html/cell-5-output-1.png" class="img-fluid figure-img"></p> | |
| </figure> | |
| </div> | |
| </div> | |
| </div> | |
| <div id="cell-8" class="cell" data-outputid="ff7c2ca8-0c6a-49d0-cfd6-4be775e012a1"> | |
| <div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>ds[<span class="st">"train"</span>][<span class="dv">0</span>][<span class="st">"image_description"</span>]</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| <div class="cell-output cell-output-display" data-execution_count="5"> | |
| <pre><code>'Two women are looking out a window. There is snow outside, and there is a snowman with human arms.'</code></pre> | |
| </div> | |
| </div> | |
| <p>We don’t have to write any function to embed examples or create an index. 🤗 datasets library’s FAISS integration abstracts these processes. We can simply use <code>map</code> method of the dataset to create a new column with the embeddings for each example like below. Let’s create one for text features on the prompt column.</p> | |
| <div id="cell-10" class="cell"> | |
| <div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> ds[<span class="st">"train"</span>]</span> | |
| <span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a>ds_with_embeddings <span class="op">=</span> dataset.<span class="bu">map</span>(<span class="kw">lambda</span> example:</span> | |
| <span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a> {<span class="st">'embeddings'</span>: model.get_text_features(</span> | |
| <span id="cb7-4"><a href="#cb7-4" aria-hidden="true" tabindex="-1"></a> <span class="op">**</span>tokenizer([example[<span class="st">"image_description"</span>]],</span> | |
| <span id="cb7-5"><a href="#cb7-5" aria-hidden="true" tabindex="-1"></a> truncation<span class="op">=</span><span class="va">True</span>, return_tensors<span class="op">=</span><span class="st">"pt"</span>)</span> | |
| <span id="cb7-6"><a href="#cb7-6" aria-hidden="true" tabindex="-1"></a> .to(<span class="st">"cuda"</span>))[<span class="dv">0</span>].detach().cpu().numpy()})</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <div id="cell-11" class="cell"> | |
| <div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>ds_with_embeddings.add_faiss_index(column<span class="op">=</span><span class="st">'embeddings'</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <p>We can do the same and get the image embeddings.</p> | |
| <div id="cell-13" class="cell"> | |
| <div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a>ds_with_embeddings <span class="op">=</span> ds_with_embeddings.<span class="bu">map</span>(<span class="kw">lambda</span> example:</span> | |
| <span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a> {<span class="st">'image_embeddings'</span>: model.get_image_features(</span> | |
| <span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a> <span class="op">**</span>processor([example[<span class="st">"image"</span>]], return_tensors<span class="op">=</span><span class="st">"pt"</span>)</span> | |
| <span id="cb9-4"><a href="#cb9-4" aria-hidden="true" tabindex="-1"></a> .to(<span class="st">"cuda"</span>))[<span class="dv">0</span>].detach().cpu().numpy()})</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <div id="cell-14" class="cell"> | |
| <div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a>ds_with_embeddings.add_faiss_index(column<span class="op">=</span><span class="st">'image_embeddings'</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <section id="querying-the-data-with-text-prompts" class="level2"> | |
| <h2 class="anchored" data-anchor-id="querying-the-data-with-text-prompts">Querying the data with text prompts</h2> | |
| <p>We can now query the dataset with text or image to get similar items from it.</p> | |
| <div id="cell-17" class="cell"> | |
| <div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a>prmt <span class="op">=</span> <span class="st">"a snowy day"</span></span> | |
| <span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a>prmt_embedding <span class="op">=</span> model.get_text_features(<span class="op">**</span>tokenizer([prmt], return_tensors<span class="op">=</span><span class="st">"pt"</span>, truncation<span class="op">=</span><span class="va">True</span>).to(<span class="st">"cuda"</span>))[<span class="dv">0</span>].detach().cpu().numpy()</span> | |
| <span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a>scores, retrieved_examples <span class="op">=</span> ds_with_embeddings.get_nearest_examples(<span class="st">'embeddings'</span>, prmt_embedding, k<span class="op">=</span><span class="dv">1</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <div id="cell-18" class="cell" data-outputid="b56009fe-dc99-4cc3-84e5-559fb3625d30"> | |
| <div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> downscale_images(image):</span> | |
| <span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a> width <span class="op">=</span> <span class="dv">200</span></span> | |
| <span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a> ratio <span class="op">=</span> (width <span class="op">/</span> <span class="bu">float</span>(image.size[<span class="dv">0</span>]))</span> | |
| <span id="cb12-4"><a href="#cb12-4" aria-hidden="true" tabindex="-1"></a> height <span class="op">=</span> <span class="bu">int</span>((<span class="bu">float</span>(image.size[<span class="dv">1</span>]) <span class="op">*</span> <span class="bu">float</span>(ratio)))</span> | |
| <span id="cb12-5"><a href="#cb12-5" aria-hidden="true" tabindex="-1"></a> img <span class="op">=</span> image.resize((width, height), Image.Resampling.LANCZOS)</span> | |
| <span id="cb12-6"><a href="#cb12-6" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> img</span> | |
| <span id="cb12-7"><a href="#cb12-7" aria-hidden="true" tabindex="-1"></a></span> | |
| <span id="cb12-8"><a href="#cb12-8" aria-hidden="true" tabindex="-1"></a>images <span class="op">=</span> [downscale_images(image) <span class="cf">for</span> image <span class="kw">in</span> retrieved_examples[<span class="st">"image"</span>]]</span> | |
| <span id="cb12-9"><a href="#cb12-9" aria-hidden="true" tabindex="-1"></a><span class="co"># see the closest text and image</span></span> | |
| <span id="cb12-10"><a href="#cb12-10" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(retrieved_examples[<span class="st">"image_description"</span>])</span> | |
| <span id="cb12-11"><a href="#cb12-11" aria-hidden="true" tabindex="-1"></a>display(images[<span class="dv">0</span>])</span> | |
| <span id="cb12-12"><a href="#cb12-12" aria-hidden="true" tabindex="-1"></a></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| <div class="cell-output cell-output-stdout"> | |
| <pre><code>['A man is in the snow. A boy with a huge snow shovel is there too. They are outside a house.']</code></pre> | |
| </div> | |
| <div class="cell-output cell-output-display"> | |
| <div> | |
| <figure class="figure"> | |
| <p><img src="faiss_files/figure-html/cell-12-output-2.png" class="img-fluid figure-img"></p> | |
| </figure> | |
| </div> | |
| </div> | |
| </div> | |
| </section> | |
| <section id="querying-the-data-with-image-prompts" class="level2"> | |
| <h2 class="anchored" data-anchor-id="querying-the-data-with-image-prompts">Querying the data with image prompts</h2> | |
| <p>Image similarity inference is similar, where you just call <code>get_image_features</code>.</p> | |
| <div id="cell-21" class="cell" data-outputid="53478699-5753-4946-90d6-0aa8b76694a6"> | |
| <div class="sourceCode cell-code" id="cb14"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb14-1"><a href="#cb14-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> requests</span> | |
| <span id="cb14-2"><a href="#cb14-2" aria-hidden="true" tabindex="-1"></a><span class="co"># image of a beaver</span></span> | |
| <span id="cb14-3"><a href="#cb14-3" aria-hidden="true" tabindex="-1"></a>url <span class="op">=</span> <span class="st">"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/beaver.png"</span></span> | |
| <span id="cb14-4"><a href="#cb14-4" aria-hidden="true" tabindex="-1"></a>image <span class="op">=</span> Image.<span class="bu">open</span>(requests.get(url, stream<span class="op">=</span><span class="va">True</span>).raw)</span> | |
| <span id="cb14-5"><a href="#cb14-5" aria-hidden="true" tabindex="-1"></a>display(downscale_images(image))</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| <div class="cell-output cell-output-display"> | |
| <div> | |
| <figure class="figure"> | |
| <p><img src="faiss_files/figure-html/cell-13-output-1.png" class="img-fluid figure-img"></p> | |
| </figure> | |
| </div> | |
| </div> | |
| </div> | |
| <p>Search for the similar image.</p> | |
| <div id="cell-23" class="cell"> | |
| <div class="sourceCode cell-code" id="cb15"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb15-1"><a href="#cb15-1" aria-hidden="true" tabindex="-1"></a>img_embedding <span class="op">=</span> model.get_image_features(<span class="op">**</span>processor([image], return_tensors<span class="op">=</span><span class="st">"pt"</span>, truncation<span class="op">=</span><span class="va">True</span>).to(<span class="st">"cuda"</span>))[<span class="dv">0</span>].detach().cpu().numpy()</span> | |
| <span id="cb15-2"><a href="#cb15-2" aria-hidden="true" tabindex="-1"></a>scores, retrieved_examples <span class="op">=</span> ds_with_embeddings.get_nearest_examples(<span class="st">'image_embeddings'</span>, img_embedding, k<span class="op">=</span><span class="dv">1</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <p>Display the most similar image to the beaver image.</p> | |
| <div id="cell-25" class="cell" data-outputid="fa620b08-4435-4929-f67f-32b3f8f46b70"> | |
| <div class="sourceCode cell-code" id="cb16"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb16-1"><a href="#cb16-1" aria-hidden="true" tabindex="-1"></a>images <span class="op">=</span> [downscale_images(image) <span class="cf">for</span> image <span class="kw">in</span> retrieved_examples[<span class="st">"image"</span>]]</span> | |
| <span id="cb16-2"><a href="#cb16-2" aria-hidden="true" tabindex="-1"></a><span class="co"># see the closest text and image</span></span> | |
| <span id="cb16-3"><a href="#cb16-3" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(retrieved_examples[<span class="st">"image_description"</span>])</span> | |
| <span id="cb16-4"><a href="#cb16-4" aria-hidden="true" tabindex="-1"></a>display(images[<span class="dv">0</span>])</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| <div class="cell-output cell-output-stdout"> | |
| <pre><code>['Salmon swim upstream but they see a grizzly bear and are in shock. The bear has a smug look on his face when he sees the salmon.']</code></pre> | |
| </div> | |
| <div class="cell-output cell-output-display"> | |
| <div> | |
| <figure class="figure"> | |
| <p><img src="faiss_files/figure-html/cell-15-output-2.png" class="img-fluid figure-img"></p> | |
| </figure> | |
| </div> | |
| </div> | |
| </div> | |
| </section> | |
| <section id="saving-pushing-and-loading-the-embeddings" class="level2"> | |
| <h2 class="anchored" data-anchor-id="saving-pushing-and-loading-the-embeddings">Saving, pushing and loading the embeddings</h2> | |
| <p>We can save the dataset with embeddings with <code>save_faiss_index</code>.</p> | |
| <div id="cell-27" class="cell"> | |
| <div class="sourceCode cell-code" id="cb18"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb18-1"><a href="#cb18-1" aria-hidden="true" tabindex="-1"></a>ds_with_embeddings.save_faiss_index(<span class="st">'embeddings'</span>, <span class="st">'embeddings/embeddings.faiss'</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <div id="cell-28" class="cell"> | |
| <div class="sourceCode cell-code" id="cb19"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb19-1"><a href="#cb19-1" aria-hidden="true" tabindex="-1"></a>ds_with_embeddings.save_faiss_index(<span class="st">'image_embeddings'</span>, <span class="st">'embeddings/image_embeddings.faiss'</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <p>It’s a good practice to store the embeddings in a dataset repository, so we will create one and push our embeddings there to pull later. We will login to Hugging Face Hub, create a dataset repository there and push our indexes there and load using <code>snapshot_download</code>.</p> | |
| <div id="cell-30" class="cell"> | |
| <div class="sourceCode cell-code" id="cb20"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb20-1"><a href="#cb20-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> huggingface_hub <span class="im">import</span> HfApi, notebook_login, snapshot_download</span> | |
| <span id="cb20-2"><a href="#cb20-2" aria-hidden="true" tabindex="-1"></a>notebook_login()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <div id="cell-31" class="cell"> | |
| <div class="sourceCode cell-code" id="cb21"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb21-1"><a href="#cb21-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> huggingface_hub <span class="im">import</span> HfApi</span> | |
| <span id="cb21-2"><a href="#cb21-2" aria-hidden="true" tabindex="-1"></a>api <span class="op">=</span> HfApi()</span> | |
| <span id="cb21-3"><a href="#cb21-3" aria-hidden="true" tabindex="-1"></a>api.create_repo(<span class="st">"merve/faiss_embeddings"</span>, repo_type<span class="op">=</span><span class="st">"dataset"</span>)</span> | |
| <span id="cb21-4"><a href="#cb21-4" aria-hidden="true" tabindex="-1"></a>api.upload_folder(</span> | |
| <span id="cb21-5"><a href="#cb21-5" aria-hidden="true" tabindex="-1"></a> folder_path<span class="op">=</span><span class="st">"./embeddings"</span>,</span> | |
| <span id="cb21-6"><a href="#cb21-6" aria-hidden="true" tabindex="-1"></a> repo_id<span class="op">=</span><span class="st">"merve/faiss_embeddings"</span>,</span> | |
| <span id="cb21-7"><a href="#cb21-7" aria-hidden="true" tabindex="-1"></a> repo_type<span class="op">=</span><span class="st">"dataset"</span>,</span> | |
| <span id="cb21-8"><a href="#cb21-8" aria-hidden="true" tabindex="-1"></a>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <div id="cell-32" class="cell"> | |
| <div class="sourceCode cell-code" id="cb22"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb22-1"><a href="#cb22-1" aria-hidden="true" tabindex="-1"></a>snapshot_download(repo_id<span class="op">=</span><span class="st">"merve/faiss_embeddings"</span>, repo_type<span class="op">=</span><span class="st">"dataset"</span>,</span> | |
| <span id="cb22-2"><a href="#cb22-2" aria-hidden="true" tabindex="-1"></a> local_dir<span class="op">=</span><span class="st">"downloaded_embeddings"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <p>We can load the embeddings to the dataset with no embeddings using <code>load_faiss_index</code>.</p> | |
| <div id="cell-34" class="cell"> | |
| <div class="sourceCode cell-code" id="cb23"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb23-1"><a href="#cb23-1" aria-hidden="true" tabindex="-1"></a>ds <span class="op">=</span> ds[<span class="st">"train"</span>]</span> | |
| <span id="cb23-2"><a href="#cb23-2" aria-hidden="true" tabindex="-1"></a>ds.load_faiss_index(<span class="st">'embeddings'</span>, <span class="st">'./downloaded_embeddings/embeddings.faiss'</span>)</span> | |
| <span id="cb23-3"><a href="#cb23-3" aria-hidden="true" tabindex="-1"></a><span class="co"># infer again</span></span> | |
| <span id="cb23-4"><a href="#cb23-4" aria-hidden="true" tabindex="-1"></a>prmt <span class="op">=</span> <span class="st">"people under the rain"</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <div id="cell-35" class="cell"> | |
| <div class="sourceCode cell-code" id="cb24"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb24-1"><a href="#cb24-1" aria-hidden="true" tabindex="-1"></a>prmt_embedding <span class="op">=</span> model.get_text_features(</span> | |
| <span id="cb24-2"><a href="#cb24-2" aria-hidden="true" tabindex="-1"></a> <span class="op">**</span>tokenizer([prmt], return_tensors<span class="op">=</span><span class="st">"pt"</span>, truncation<span class="op">=</span><span class="va">True</span>)</span> | |
| <span id="cb24-3"><a href="#cb24-3" aria-hidden="true" tabindex="-1"></a> .to(<span class="st">"cuda"</span>))[<span class="dv">0</span>].detach().cpu().numpy()</span> | |
| <span id="cb24-4"><a href="#cb24-4" aria-hidden="true" tabindex="-1"></a></span> | |
| <span id="cb24-5"><a href="#cb24-5" aria-hidden="true" tabindex="-1"></a>scores, retrieved_examples <span class="op">=</span> ds.get_nearest_examples(<span class="st">'embeddings'</span>, prmt_embedding, k<span class="op">=</span><span class="dv">1</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| </div> | |
| <div id="cell-36" class="cell" data-outputid="8d5008b4-ab8f-4b42-92e7-b29e57c126cb"> | |
| <div class="sourceCode cell-code" id="cb25"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb25-1"><a href="#cb25-1" aria-hidden="true" tabindex="-1"></a>display(retrieved_examples[<span class="st">"image"</span>][<span class="dv">0</span>])</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div> | |
| <div class="cell-output cell-output-display"> | |
| <div> | |
| <figure class="figure"> | |
| <p><img src="faiss_files/figure-html/cell-23-output-1.png" class="img-fluid figure-img"></p> | |
| </figure> | |
| </div> | |
| </div> | |
| </div> | |
| </section> | |
| </section> | |
| </main> <!-- /main --> | |
| <script id="quarto-html-after-body" type="application/javascript"> | |
| window.document.addEventListener("DOMContentLoaded", function (event) { | |
| const toggleBodyColorMode = (bsSheetEl) => { | |
| const mode = bsSheetEl.getAttribute("data-mode"); | |
| const bodyEl = window.document.querySelector("body"); | |
| if (mode === "dark") { | |
| bodyEl.classList.add("quarto-dark"); | |
| bodyEl.classList.remove("quarto-light"); | |
| } else { | |
| bodyEl.classList.add("quarto-light"); | |
| bodyEl.classList.remove("quarto-dark"); | |
| } | |
| } | |
| const toggleBodyColorPrimary = () => { | |
| const bsSheetEl = window.document.querySelector("link#quarto-bootstrap"); | |
| if (bsSheetEl) { | |
| toggleBodyColorMode(bsSheetEl); | |
| } | |
| } | |
| toggleBodyColorPrimary(); | |
| const icon = ""; | |
| const anchorJS = new window.AnchorJS(); | |
| anchorJS.options = { | |
| placement: 'right', | |
| icon: icon | |
| }; | |
| anchorJS.add('.anchored'); | |
| const isCodeAnnotation = (el) => { | |
| for (const clz of el.classList) { | |
| if (clz.startsWith('code-annotation-')) { | |
| return true; | |
| } | |
| } | |
| return false; | |
| } | |
| const onCopySuccess = function(e) { | |
| // button target | |
| const button = e.trigger; | |
| // don't keep focus | |
| button.blur(); | |
| // flash "checked" | |
| button.classList.add('code-copy-button-checked'); | |
| var currentTitle = button.getAttribute("title"); | |
| button.setAttribute("title", "Copied!"); | |
| let tooltip; | |
| if (window.bootstrap) { | |
| button.setAttribute("data-bs-toggle", "tooltip"); | |
| button.setAttribute("data-bs-placement", "left"); | |
| button.setAttribute("data-bs-title", "Copied!"); | |
| tooltip = new bootstrap.Tooltip(button, | |
| { trigger: "manual", | |
| customClass: "code-copy-button-tooltip", | |
| offset: [0, -8]}); | |
| tooltip.show(); | |
| } | |
| setTimeout(function() { | |
| if (tooltip) { | |
| tooltip.hide(); | |
| button.removeAttribute("data-bs-title"); | |
| button.removeAttribute("data-bs-toggle"); | |
| button.removeAttribute("data-bs-placement"); | |
| } | |
| button.setAttribute("title", currentTitle); | |
| button.classList.remove('code-copy-button-checked'); | |
| }, 1000); | |
| // clear code selection | |
| e.clearSelection(); | |
| } | |
| const getTextToCopy = function(trigger) { | |
| const codeEl = trigger.previousElementSibling.cloneNode(true); | |
| for (const childEl of codeEl.children) { | |
| if (isCodeAnnotation(childEl)) { | |
| childEl.remove(); | |
| } | |
| } | |
| return codeEl.innerText; | |
| } | |
| const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', { | |
| text: getTextToCopy | |
| }); | |
| clipboard.on('success', onCopySuccess); | |
| if (window.document.getElementById('quarto-embedded-source-code-modal')) { | |
| const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', { | |
| text: getTextToCopy, | |
| container: window.document.getElementById('quarto-embedded-source-code-modal') | |
| }); | |
| clipboardModal.on('success', onCopySuccess); | |
| } | |
| var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//); | |
| var mailtoRegex = new RegExp(/^mailto:/); | |
| var filterRegex = new RegExp('/' + window.location.host + '/'); | |
| var isInternal = (href) => { | |
| return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href); | |
| } | |
| // Inspect non-navigation links and adorn them if external | |
| var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)'); | |
| for (var i=0; i<links.length; i++) { | |
| const link = links[i]; | |
| if (!isInternal(link.href)) { | |
| // undo the damage that might have been done by quarto-nav.js in the case of | |
| // links that we want to consider external | |
| if (link.dataset.originalHref !== undefined) { | |
| link.href = link.dataset.originalHref; | |
| } | |
| } | |
| } | |
| function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) { | |
| const config = { | |
| allowHTML: true, | |
| maxWidth: 500, | |
| delay: 100, | |
| arrow: false, | |
| appendTo: function(el) { | |
| return el.parentElement; | |
| }, | |
| interactive: true, | |
| interactiveBorder: 10, | |
| theme: 'quarto', | |
| placement: 'bottom-start', | |
| }; | |
| if (contentFn) { | |
| config.content = contentFn; | |
| } | |
| if (onTriggerFn) { | |
| config.onTrigger = onTriggerFn; | |
| } | |
| if (onUntriggerFn) { | |
| config.onUntrigger = onUntriggerFn; | |
| } | |
| window.tippy(el, config); | |
| } | |
| const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]'); | |
| for (var i=0; i<noterefs.length; i++) { | |
| const ref = noterefs[i]; | |
| tippyHover(ref, function() { | |
| // use id or data attribute instead here | |
| let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href'); | |
| try { href = new URL(href).hash; } catch {} | |
| const id = href.replace(/^#\/?/, ""); | |
| const note = window.document.getElementById(id); | |
| if (note) { | |
| return note.innerHTML; | |
| } else { | |
| return ""; | |
| } | |
| }); | |
| } | |
| const xrefs = window.document.querySelectorAll('a.quarto-xref'); | |
| const processXRef = (id, note) => { | |
| // Strip column container classes | |
| const stripColumnClz = (el) => { | |
| el.classList.remove("page-full", "page-columns"); | |
| if (el.children) { | |
| for (const child of el.children) { | |
| stripColumnClz(child); | |
| } | |
| } | |
| } | |
| stripColumnClz(note) | |
| if (id === null || id.startsWith('sec-')) { | |
| // Special case sections, only their first couple elements | |
| const container = document.createElement("div"); | |
| if (note.children && note.children.length > 2) { | |
| container.appendChild(note.children[0].cloneNode(true)); | |
| for (let i = 1; i < note.children.length; i++) { | |
| const child = note.children[i]; | |
| if (child.tagName === "P" && child.innerText === "") { | |
| continue; | |
| } else { | |
| container.appendChild(child.cloneNode(true)); | |
| break; | |
| } | |
| } | |
| if (window.Quarto?.typesetMath) { | |
| window.Quarto.typesetMath(container); | |
| } | |
| return container.innerHTML | |
| } else { | |
| if (window.Quarto?.typesetMath) { | |
| window.Quarto.typesetMath(note); | |
| } | |
| return note.innerHTML; | |
| } | |
| } else { | |
| // Remove any anchor links if they are present | |
| const anchorLink = note.querySelector('a.anchorjs-link'); | |
| if (anchorLink) { | |
| anchorLink.remove(); | |
| } | |
| if (window.Quarto?.typesetMath) { | |
| window.Quarto.typesetMath(note); | |
| } | |
| if (note.classList.contains("callout")) { | |
| return note.outerHTML; | |
| } else { | |
| return note.innerHTML; | |
| } | |
| } | |
| } | |
| for (var i=0; i<xrefs.length; i++) { | |
| const xref = xrefs[i]; | |
| tippyHover(xref, undefined, function(instance) { | |
| instance.disable(); | |
| let url = xref.getAttribute('href'); | |
| let hash = undefined; | |
| if (url.startsWith('#')) { | |
| hash = url; | |
| } else { | |
| try { hash = new URL(url).hash; } catch {} | |
| } | |
| if (hash) { | |
| const id = hash.replace(/^#\/?/, ""); | |
| const note = window.document.getElementById(id); | |
| if (note !== null) { | |
| try { | |
| const html = processXRef(id, note.cloneNode(true)); | |
| instance.setContent(html); | |
| } finally { | |
| instance.enable(); | |
| instance.show(); | |
| } | |
| } else { | |
| // See if we can fetch this | |
| fetch(url.split('#')[0]) | |
| .then(res => res.text()) | |
| .then(html => { | |
| const parser = new DOMParser(); | |
| const htmlDoc = parser.parseFromString(html, "text/html"); | |
| const note = htmlDoc.getElementById(id); | |
| if (note !== null) { | |
| const html = processXRef(id, note); | |
| instance.setContent(html); | |
| } | |
| }).finally(() => { | |
| instance.enable(); | |
| instance.show(); | |
| }); | |
| } | |
| } else { | |
| // See if we can fetch a full url (with no hash to target) | |
| // This is a special case and we should probably do some content thinning / targeting | |
| fetch(url) | |
| .then(res => res.text()) | |
| .then(html => { | |
| const parser = new DOMParser(); | |
| const htmlDoc = parser.parseFromString(html, "text/html"); | |
| const note = htmlDoc.querySelector('main.content'); | |
| if (note !== null) { | |
| // This should only happen for chapter cross references | |
| // (since there is no id in the URL) | |
| // remove the first header | |
| if (note.children.length > 0 && note.children[0].tagName === "HEADER") { | |
| note.children[0].remove(); | |
| } | |
| const html = processXRef(null, note); | |
| instance.setContent(html); | |
| } | |
| }).finally(() => { | |
| instance.enable(); | |
| instance.show(); | |
| }); | |
| } | |
| }, function(instance) { | |
| }); | |
| } | |
| let selectedAnnoteEl; | |
| const selectorForAnnotation = ( cell, annotation) => { | |
| let cellAttr = 'data-code-cell="' + cell + '"'; | |
| let lineAttr = 'data-code-annotation="' + annotation + '"'; | |
| const selector = 'span[' + cellAttr + '][' + lineAttr + ']'; | |
| return selector; | |
| } | |
| const selectCodeLines = (annoteEl) => { | |
| const doc = window.document; | |
| const targetCell = annoteEl.getAttribute("data-target-cell"); | |
| const targetAnnotation = annoteEl.getAttribute("data-target-annotation"); | |
| const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation)); | |
| const lines = annoteSpan.getAttribute("data-code-lines").split(","); | |
| const lineIds = lines.map((line) => { | |
| return targetCell + "-" + line; | |
| }) | |
| let top = null; | |
| let height = null; | |
| let parent = null; | |
| if (lineIds.length > 0) { | |
| //compute the position of the single el (top and bottom and make a div) | |
| const el = window.document.getElementById(lineIds[0]); | |
| top = el.offsetTop; | |
| height = el.offsetHeight; | |
| parent = el.parentElement.parentElement; | |
| if (lineIds.length > 1) { | |
| const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]); | |
| const bottom = lastEl.offsetTop + lastEl.offsetHeight; | |
| height = bottom - top; | |
| } | |
| if (top !== null && height !== null && parent !== null) { | |
| // cook up a div (if necessary) and position it | |
| let div = window.document.getElementById("code-annotation-line-highlight"); | |
| if (div === null) { | |
| div = window.document.createElement("div"); | |
| div.setAttribute("id", "code-annotation-line-highlight"); | |
| div.style.position = 'absolute'; | |
| parent.appendChild(div); | |
| } | |
| div.style.top = top - 2 + "px"; | |
| div.style.height = height + 4 + "px"; | |
| div.style.left = 0; | |
| let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter"); | |
| if (gutterDiv === null) { | |
| gutterDiv = window.document.createElement("div"); | |
| gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter"); | |
| gutterDiv.style.position = 'absolute'; | |
| const codeCell = window.document.getElementById(targetCell); | |
| const gutter = codeCell.querySelector('.code-annotation-gutter'); | |
| gutter.appendChild(gutterDiv); | |
| } | |
| gutterDiv.style.top = top - 2 + "px"; | |
| gutterDiv.style.height = height + 4 + "px"; | |
| } | |
| selectedAnnoteEl = annoteEl; | |
| } | |
| }; | |
| const unselectCodeLines = () => { | |
| const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"]; | |
| elementsIds.forEach((elId) => { | |
| const div = window.document.getElementById(elId); | |
| if (div) { | |
| div.remove(); | |
| } | |
| }); | |
| selectedAnnoteEl = undefined; | |
| }; | |
| // Handle positioning of the toggle | |
| window.addEventListener( | |
| "resize", | |
| throttle(() => { | |
| elRect = undefined; | |
| if (selectedAnnoteEl) { | |
| selectCodeLines(selectedAnnoteEl); | |
| } | |
| }, 10) | |
| ); | |
| function throttle(fn, ms) { | |
| let throttle = false; | |
| let timer; | |
| return (...args) => { | |
| if(!throttle) { // first call gets through | |
| fn.apply(this, args); | |
| throttle = true; | |
| } else { // all the others get throttled | |
| if(timer) clearTimeout(timer); // cancel #2 | |
| timer = setTimeout(() => { | |
| fn.apply(this, args); | |
| timer = throttle = false; | |
| }, ms); | |
| } | |
| }; | |
| } | |
| // Attach click handler to the DT | |
| const annoteDls = window.document.querySelectorAll('dt[data-target-cell]'); | |
| for (const annoteDlNode of annoteDls) { | |
| annoteDlNode.addEventListener('click', (event) => { | |
| const clickedEl = event.target; | |
| if (clickedEl !== selectedAnnoteEl) { | |
| unselectCodeLines(); | |
| const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active'); | |
| if (activeEl) { | |
| activeEl.classList.remove('code-annotation-active'); | |
| } | |
| selectCodeLines(clickedEl); | |
| clickedEl.classList.add('code-annotation-active'); | |
| } else { | |
| // Unselect the line | |
| unselectCodeLines(); | |
| clickedEl.classList.remove('code-annotation-active'); | |
| } | |
| }); | |
| } | |
| const findCites = (el) => { | |
| const parentEl = el.parentElement; | |
| if (parentEl) { | |
| const cites = parentEl.dataset.cites; | |
| if (cites) { | |
| return { | |
| el, | |
| cites: cites.split(' ') | |
| }; | |
| } else { | |
| return findCites(el.parentElement) | |
| } | |
| } else { | |
| return undefined; | |
| } | |
| }; | |
| var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]'); | |
| for (var i=0; i<bibliorefs.length; i++) { | |
| const ref = bibliorefs[i]; | |
| const citeInfo = findCites(ref); | |
| if (citeInfo) { | |
| tippyHover(citeInfo.el, function() { | |
| var popup = window.document.createElement('div'); | |
| citeInfo.cites.forEach(function(cite) { | |
| var citeDiv = window.document.createElement('div'); | |
| citeDiv.classList.add('hanging-indent'); | |
| citeDiv.classList.add('csl-entry'); | |
| var biblioDiv = window.document.getElementById('ref-' + cite); | |
| if (biblioDiv) { | |
| citeDiv.innerHTML = biblioDiv.innerHTML; | |
| } | |
| popup.appendChild(citeDiv); | |
| }); | |
| return popup.innerHTML; | |
| }); | |
| } | |
| } | |
| }); | |
| </script> | |
| </div> <!-- /content --> | |
| </body></html> |