updates

README.md

@@ -1,13 +1,14 @@
 ---
-title: marimo app template
-emoji: 🍃
-colorFrom: indigo
-colorTo: purple
+title: motherduck embedding explorer
+emoji: 🦆
+colorFrom: purple
+colorTo: indigo
 sdk: docker
-pinned: true
+pinned: false
 license: mit
-short_description: Template for deploying a marimo application to HF
+short_description: motherduck embedding explorer
 ---
 
 Check out marimo at <https://github.com/marimo-team/marimo>
+Check out motherduck at <https://motherduck.com>
 Check out the configuration reference at <https://huggingface.co/docs/hub/spaces-config-reference>

app.py

@@ -1,469 +1,292 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+#     "altair==5.4.1",
+#     "duckdb==1.1.3",
+#     "hdbscan==0.8.39",
+#     "marimo",
+#     "numba==0.60.0",
+#     "numpy==2.0.2",
+#     "polars==1.17.1",
+#     "pyarrow==18.0.0",
+#     "scikit-learn==1.5.2",
+#     "umap-learn==0.5.7",
+# ]
+# ///
+
 import marimo
 
-__generated_with = "0.9.2"
-app = marimo.App()
+__generated_with = "0.9.33"
+app = marimo.App(width="medium")
 
 
 @app.cell
 def __():
     import marimo as mo
 
-    mo.md("# Welcome to marimo! 🌊🍃")
     return (mo,)
 
 
-@app.cell
+@app.cell(hide_code=True)
 def __(mo):
-    slider = mo.ui.slider(1, 22)
-    return (slider,)
-
-
-@app.cell
-def __(mo, slider):
     mo.md(
-        f"""
-        marimo is a **reactive** Python notebook.
+        r"""
+        # Visualizing text embeddings using MotherDuck and marimo
 
-        This means that unlike traditional notebooks, marimo notebooks **run
-        automatically** when you modify them or
-        interact with UI elements, like this slider: {slider}.
+        > Text embeddings have become a crucial tool in AI/ML applications, allowing us to convert text into numerical vectors that capture semantic meaning. These vectors are often used for semantic search, but in this blog post, we'll explore how to visualize and explore text embeddings interactively using MotherDuck and marimo.
 
-        {"##" + "🍃" * slider.value}
+        [_Read the full blog here._](https://motherduck.com/blog/MotherDuck-Visualize-Embeddings-Marimo/)
         """
     )
     return
 
 
-@app.cell(hide_code=True)
-def __(mo):
-    mo.accordion(
-        {
-            "Tip: disabling automatic execution": mo.md(
-                rf"""
-            marimo lets you disable automatic execution: just go into the
-            notebook settings and set
-
-            "Runtime > On Cell Change" to "lazy".
-
-            When the runtime is lazy, after running a cell, marimo marks its
-            descendants as stale instead of automatically running them. The
-            lazy runtime puts you in control over when cells are run, while
-            still giving guarantees about the notebook state.
-            """
-            )
-        }
-    )
-    return
-
-
-@app.cell(hide_code=True)
-def __(mo):
-    mo.md(
-        """
-        Tip: This is a tutorial notebook. You can create your own notebooks
-        by entering `marimo edit` at the command line.
-        """
-    ).callout()
-    return
-
-
 @app.cell(hide_code=True)
 def __(mo):
     mo.md(
         """
-        ## 1. Reactive execution
-
-        A marimo notebook is made up of small blocks of Python code called
-        cells.
+        ## Connecting to MotherDuck and Loading Sample Data
 
-        marimo reads your cells and models the dependencies among them: whenever
-        a cell that defines a global variable  is run, marimo
-        **automatically runs** all cells that reference that variable.
+        This data has already been pre-computed, but you can fork and edit this notebook to run with your own data!
 
-        Reactivity keeps your program state and outputs in sync with your code,
-        making for a dynamic programming environment that prevents bugs before they
-        happen.
+        ```sql
+        ATTACH IF NOT EXISTS 'md:my_db'
+        SELECT * FROM my_db.demo_with_embeddings;
+        ```
         """
     )
     return
 
 
-@app.cell(hide_code=True)
-def __(changed, mo):
-    (
-        mo.md(
-            f"""
-            **✨ Nice!** The value of `changed` is now {changed}.
-
-            When you updated the value of the variable `changed`, marimo
-            **reacted** by running this cell automatically, because this cell
-            references the global variable `changed`.
-
-            Reactivity ensures that your notebook state is always
-            consistent, which is crucial for doing good science; it's also what
-            enables marimo notebooks to double as tools and  apps.
-            """
-        )
-        if changed
-        else mo.md(
-            """
-            **🌊 See it in action.** In the next cell, change the value of the
-            variable  `changed` to `True`, then click the run button.
-            """
-        )
-    )
-    return
-
-
 @app.cell
-def __():
-    changed = False
-    return (changed,)
-
-
-@app.cell(hide_code=True)
 def __(mo):
-    mo.accordion(
-        {
-            "Tip: execution order": (
-                """
-                The order of cells on the page has no bearing on
-                the order in which cells are executed: marimo knows that a cell
-                reading a variable must run after the cell that  defines it. This
-                frees you to organize your code in the way that makes the most
-                sense for you.
-                """
-            )
-        }
-    )
-    return
-
-
-@app.cell(hide_code=True)
-def __(mo):
-    mo.md(
+    _df = mo.sql(
         """
-        **Global names must be unique.** To enable reactivity, marimo imposes a
-        constraint on how names appear in cells: no two cells may define the same
-        variable.
+        ATTACH IF NOT EXISTS 'md:my_db'
         """
     )
-    return
+    return (my_db,)
 
 
-@app.cell(hide_code=True)
-def __(mo):
-    mo.accordion(
-        {
-            "Tip: encapsulation": (
-                """
-                By encapsulating logic in functions, classes, or Python modules,
-                you can minimize the number of global variables in your notebook.
-                """
-            )
-        }
-    )
-    return
-
-
-@app.cell(hide_code=True)
-def __(mo):
-    mo.accordion(
-        {
-            "Tip: private variables": (
-                """
-                Variables prefixed with an underscore are "private" to a cell, so
-                they can be defined by multiple cells.
-                """
-            )
-        }
-    )
-    return
-
-
-@app.cell(hide_code=True)
+@app.cell
 def __(mo):
-    mo.md(
+    _df = mo.sql(
         """
-        ## 2. UI elements
-
-        Cells can output interactive UI elements. Interacting with a UI
-        element **automatically triggers notebook execution**: when
-        you interact with a UI element, its value is sent back to Python, and
-        every cell that references that element is re-run.
-
-        marimo provides a library of UI elements to choose from under
-        `marimo.ui`.
+        -- Commented out as we have already run the embeddings for showcasing purposes.
+
+        -- CREATE OR REPLACE TABLE my_db.demo_embedding_data AS
+        -- SELECT DISTINCT ON (url) *  -- Remove duplicate URLs
+        -- FROM 'hf://datasets/julien040/hacker-news-posts/story.parquet'
+        -- WHERE contains(title, 'database')  -- Filter for posts about databases
+        --     AND score > 5  -- Only include popular posts
+        -- LIMIT 50000;
         """
     )
     return
 
 
 @app.cell
-def __(mo):
-    mo.md("""**🌊 Some UI elements.** Try interacting with the below elements.""")
-    return
-
+def __(demo_with_embeddings, mo, my_db):
+    embeddings = mo.sql(
+        f"""
+        -- Commented out as we have already run the embeddings for showcasing purposes.
+        -- CREATE TABLE my_db.demo_with_embeddings AS
+        -- SELECT *, embedding(title) as text_embedding
+        -- FROM my_db.demo_embedding_data
+        -- LIMIT 1500;
 
-@app.cell
-def __(mo):
-    icon = mo.ui.dropdown(["🍃", "🌊", "✨"], value="🍃")
-    return (icon,)
+        SELECT title, text_embedding, * EXCLUDE(id, title, text_embedding, comments) FROM my_db.demo_with_embeddings;
+        """
+    )
+    return (embeddings,)
 
 
 @app.cell
-def __(icon, mo):
-    repetitions = mo.ui.slider(1, 16, label=f"number of {icon.value}: ")
-    return (repetitions,)
+def __(PCA, hdbscan, np, umap):
+    def umap_reduce(np_array, metric="cosine"):
+        """
+        Reduce the dimensionality of the embeddings to 2D using
+        UMAP algorithm. UMAP preserves both local and global structure
+        of the high-dimensional data.
+        """
+        reducer = umap.UMAP(
+            n_components=2,  # Reduce to 2D for visualization
+            metric=metric,  # Default: cosine similarity for text embeddings
+            n_neighbors=80,  # Higher values = more global structure
+            min_dist=0.1,  # Controls how tightly points cluster
+        )
+        return reducer.fit_transform(np_array)
 
+    def cluster_points(np_array, min_cluster_size=4, max_cluster_size=50):
+        """
+        Cluster the embeddings using HDBSCAN algorithm.
+        We first reduce dimensionality to 50D with PCA to speed up clustering,
+        while still preserving most of the important information.
+        """
+        pca = PCA(n_components=50)
+        np_array = pca.fit_transform(np_array)
 
-@app.cell
-def __(icon, repetitions):
-    icon, repetitions
-    return
+        hdb = hdbscan.HDBSCAN(
+            min_samples=3,  # Minimum points to form dense region
+            min_cluster_size=min_cluster_size,  # Minimum size of a cluster
+            max_cluster_size=max_cluster_size,  # Maximum size of a cluster
+        ).fit(np_array)
 
+        return np.where(
+            hdb.labels_ == -1, "outlier", "cluster_" + hdb.labels_.astype(str)
+        )
 
-@app.cell
-def __(icon, mo, repetitions):
-    mo.md("# " + icon.value * repetitions.value)
-    return
+    return cluster_points, umap_reduce
 
 
-@app.cell(hide_code=True)
+@app.cell
 def __(mo):
-    mo.md(
-        """
-        ## 3. marimo is just Python
-
-        marimo cells parse Python (and only Python), and marimo notebooks are
-        stored as pure Python files — outputs are _not_ included. There's no
-        magical syntax.
-
-        The Python files generated by marimo are:
-
-        - easily versioned with git, yielding minimal diffs
-        - legible for both humans and machines
-        - formattable using your tool of choice,
-        - usable as Python  scripts, with UI  elements taking their default
-        values, and
-        - importable by other modules (more on that in the future).
-        """
+    cluster_size_slider = mo.ui.range_slider(
+        start=1,
+        stop=80,
+        value=(4, 50),
+        step=1,
+        show_value=True,
+        debounce=True,
+        label="Cluster Size (min, max)",
     )
-    return
+    metric_dropdown = mo.ui.dropdown(
+        ["cosine", "euclidean", "manhattan", "mahalanobis"],
+        value="cosine",
+        label="Distance Metric",
+    )
+    return cluster_size_slider, metric_dropdown
 
 
-@app.cell(hide_code=True)
+@app.cell
 def __(mo):
     mo.md(
-        """
-        ## 4. Running notebooks as apps
-
-        marimo notebooks can double as apps. Click the app window icon in the
-        bottom-right to see this notebook in "app view."
+        r"""
+        ## Processing the Data
 
-        Serve a notebook as an app with `marimo run` at the command-line.
-        Of course, you can use marimo just to level-up your
-        notebooking, without ever making apps.
+        Now we'll transform our high-dimensional embeddings into something we can visualize, using `umap_reduce` and `cluster_points`. More details on this step [in the blog](https://motherduck.com/blog/MotherDuck-Visualize-Embeddings-Marimo/).
         """
     )
     return
 
 
-@app.cell(hide_code=True)
-def __(mo):
-    mo.md(
-        """
-        ## 5. The `marimo` command-line tool
-
-        **Creating and editing notebooks.** Use
-
-        ```
-        marimo edit
-        ```
-
-        in a terminal to start the marimo notebook server. From here
-        you can create a new notebook or edit existing ones.
-
-
-        **Running as apps.** Use
+@app.cell
+def __(
+    cluster_points,
+    cluster_size_slider,
+    embeddings,
+    metric_dropdown,
+    mo,
+    umap_reduce,
+):
+    with mo.status.spinner("Clustering points...") as _s:
+        embeddings_array = embeddings["text_embedding"].to_numpy()
+        hdb_labels = cluster_points(
+            embeddings_array,
+            min_cluster_size=cluster_size_slider.value[0],
+            max_cluster_size=cluster_size_slider.value[1],
+        )
+        _s.update("Reducing dimensionality...")
+        embeddings_2d = umap_reduce(embeddings_array, metric=metric_dropdown.value)
+    mo.show_code()
+    return embeddings_2d, embeddings_array, hdb_labels
 
-        ```
-        marimo run notebook.py
-        ```
 
-        to start a webserver that serves your notebook as an app in read-only mode,
-        with code cells hidden.
+@app.cell
+def __(cluster_size_slider, metric_dropdown, mo):
+    mo.hstack([cluster_size_slider, metric_dropdown])
+    return
 
-        **Convert a Jupyter notebook.** Convert a Jupyter notebook to a marimo
-        notebook using `marimo convert`:
 
-        ```
-        marimo convert your_notebook.ipynb > your_app.py
-        ```
-
-        **Tutorials.** marimo comes packaged with tutorials:
+@app.cell
+def __(embeddings, embeddings_2d, hdb_labels, pl):
+    data = embeddings.lazy()  # Lazy evaluation for performance
+    data = data.with_columns(
+        text_embedding_2d_1=embeddings_2d[:, 0],
+        text_embedding_2d_2=embeddings_2d[:, 1],
+        cluster=hdb_labels,
+    )
+    data = data.unique(subset=["url"], maintain_order=True)  # Remove duplicate URLs
+    data = data.drop(["text_embedding"])  # Drop unused columns
+    data = data.filter(pl.col("cluster") != "outlier")  # Filter out outliers
+    data = data.collect()  # Collect the data
+    return (data,)
 
-        - `dataflow`: more on marimo's automatic execution
-        - `ui`: how to use UI elements
-        - `markdown`: how to write markdown, with interpolated values and
-           LaTeX
-        - `plots`: how plotting works in marimo
-        - `sql`: how to use SQL
-        - `layout`: layout elements in marimo
-        - `fileformat`: how marimo's file format works
-        - `markdown-format`: for using `.md` files in marimo
-        - `for-jupyter-users`: if you are coming from Jupyter
 
-        Start a tutorial with `marimo tutorial`; for example,
+@app.cell
+def __(data):
+    data.select(
+        "title", "cluster", "text_embedding_2d_1", "text_embedding_2d_2", "score"
+    )
+    return
 
-        ```
-        marimo tutorial dataflow
-        ```
 
-        In addition to tutorials, we have examples in our
-        [our GitHub repo](https://www.github.com/marimo-team/marimo/tree/main/examples).
-        """
+@app.cell
+def __(alt, data, mo):
+    chart = (
+        alt.Chart(data)
+        .mark_point()
+        .encode(
+            x=alt.X("text_embedding_2d_1").scale(zero=False),
+            y=alt.Y("text_embedding_2d_2").scale(zero=False),
+            color="cluster",
+            tooltip=["title", "score", "cluster"],
+        )
     )
-    return
+    chart = mo.ui.altair_chart(chart)
+    mo.show_code()
+    return (chart,)
 
 
 @app.cell(hide_code=True)
 def __(mo):
     mo.md(
-        """
-        ## 6. The marimo editor
+        r"""
+        ## Creating an Interactive Visualization
 
-        Here are some tips to help you get started with the marimo editor.
+        We will plot the 2D representation of the text embeddings, colored by the clusters identified by HDBSCAN. You can select points on the chart to explore the text embeddings further. 👇
         """
     )
     return
 
 
 @app.cell
-def __(mo, tips):
-    mo.accordion(tips)
+def __(chart):
+    chart
     return
 
 
-@app.cell(hide_code=True)
-def __(mo):
-    mo.md("""## Finally, a fun fact""")
+@app.cell
+def __(chart):
+    chart.value
     return
 
 
-@app.cell(hide_code=True)
+@app.cell
 def __(mo):
-    mo.md(
-        """
-        The name "marimo" is a reference to a type of algae that, under
-        the right conditions, clumps together to form a small sphere
-        called a "marimo moss ball". Made of just strands of algae, these
-        beloved assemblages are greater than the sum of their parts.
-        """
-    )
+    # Empty space for the table
+    mo.Html("<div style='height: 400px;'></div>")
     return
 
 
-@app.cell(hide_code=True)
+@app.cell
 def __():
-    tips = {
-        "Saving": (
-            """
-            **Saving**
-
-            - _Name_ your app using the box at the top of the screen, or
-              with `Ctrl/Cmd+s`. You can also create a named app at the
-              command line, e.g., `marimo edit app_name.py`.
-
-            - _Save_ by clicking the save icon on the bottom right, or by
-              inputting `Ctrl/Cmd+s`. By default marimo is configured
-              to autosave.
-            """
-        ),
-        "Running": (
-            """
-            1. _Run a cell_ by clicking the play ( ▷ ) button on the top
-            right of a cell, or by inputting `Ctrl/Cmd+Enter`.
-
-            2. _Run a stale cell_  by clicking the yellow run button on the
-            right of the cell, or by inputting `Ctrl/Cmd+Enter`. A cell is
-            stale when its code has been modified but not run.
-
-            3. _Run all stale cells_ by clicking the play ( ▷ ) button on
-            the bottom right of the screen, or input `Ctrl/Cmd+Shift+r`.
-            """
-        ),
-        "Console Output": (
-            """
-            Console output (e.g., `print()` statements) is shown below a
-            cell.
-            """
-        ),
-        "Creating, Moving, and Deleting Cells": (
-            """
-            1. _Create_ a new cell above or below a given one by clicking
-                the plus button to the left of the cell, which appears on
-                mouse hover.
-
-            2. _Move_ a cell up or down by dragging on the handle to the 
-                right of the cell, which appears on mouse hover.
-
-            3. _Delete_ a cell by clicking the trash bin icon. Bring it
-                back by clicking the undo button on the bottom right of the
-                screen, or with `Ctrl/Cmd+Shift+z`.
-            """
-        ),
-        "Disabling Automatic Execution": (
-            """
-            Via the notebook settings (gear icon) or footer panel, you
-            can disable automatic execution. This is helpful when
-            working with expensive notebooks or notebooks that have
-            side-effects like database transactions.
-            """
-        ),
-        "Disabling Cells": (
-            """
-            You can disable a cell via the cell context menu.
-            marimo will never run a disabled cell or any cells that depend on it.
-            This can help prevent accidental execution of expensive computations
-            when editing a notebook.
-            """
-        ),
-        "Code Folding": (
-            """
-            You can collapse or fold the code in a cell by clicking the arrow
-            icons in the line number column to the left, or by using keyboard
-            shortcuts.
-
-            Use the command palette (`Ctrl/Cmd+k`) or a keyboard shortcut to
-            quickly fold or unfold all cells.
-            """
-        ),
-        "Code Formatting": (
-            """
-            If you have [ruff](https://github.com/astral-sh/ruff) installed,
-            you can format a cell with the keyboard shortcut `Ctrl/Cmd+b`.
-            """
-        ),
-        "Command Palette": (
-            """
-            Use `Ctrl/Cmd+k` to open the command palette.
-            """
-        ),
-        "Keyboard Shortcuts": (
-            """
-            Open the notebook menu (top-right) or input `Ctrl/Cmd+Shift+h` to
-            view a list of all keyboard shortcuts.
-            """
-        ),
-        "Configuration": (
-            """
-           Configure the editor by clicking the gears icon near the top-right
-           of the screen.
-           """
-        ),
-    }
-    return (tips,)
+    # Data manipulation and database connections
+    import polars as pl
+    import duckdb
+    import numba  # <- FYI, this module takes a while to load, be patient
+    import pyarrow
+
+    # Visualization
+    import altair as alt
+
+    # ML tools for dimensionality reduction and clustering
+    import umap  # For reducing high-dimensional embeddings to 2D
+    import hdbscan  # For clustering similar embeddings
+    import numpy as np
+    from sklearn.decomposition import PCA
+
+    return PCA, alt, duckdb, hdbscan, np, numba, pl, pyarrow, umap
 
 
 if __name__ == "__main__":

requirements.txt

@@ -1,5 +1,10 @@
+altair==5.4.1
+duckdb==1.1.3
+hdbscan==0.8.39
 marimo
-# Or a specific version
-# marimo>=0.9.0
-
-# Add other dependencies as needed
+numba==0.60.0
+numpy==2.0.2
+polars==1.17.1
+pyarrow==18.0.0
+scikit-learn==1.5.2
+umap-learn==0.5.7