Duplicate from keturn/INED-datasette

Co-authored-by: Kevin Turner <[email protected]>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,31 @@
+FROM datasetteproject/datasette:0.64.1
+
+# huggingface spaces run as user 1000
+RUN adduser hf-space --uid 1000 --disabled-password --gecos '' && \
+    mkdir /home/hf-space/app && \
+    chown hf-space: /home/hf-space/app
+WORKDIR /home/hf-space/app
+
+RUN datasette install datasette-configure-fts && \
+    datasette install datasette-render-image-tags
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git && \
+    apt-get clean && \
+    rm -rf /var/lib/apt && \
+    rm -rf /var/lib/dpkg/info/*
+
+USER hf-space
+
+# spaces default port
+EXPOSE 7860
+ENTRYPOINT ["datasette", "--host=0.0.0.0", "--port=7860"]
+CMD ["."]
+
+ENV PYTHONPATH=/home/hf-space/app/src/
+
+COPY src src
+COPY metadata.json settings.json ./
+
+RUN src/import-git.sh && \
+    datasette inspect *.db --inspect-file=inspect-data.json

README.md

@@ -0,0 +1,11 @@
+---
+title: INED Datasette
+emoji: 🐢
+colorFrom: indigo
+colorTo: pink
+sdk: docker
+pinned: false
+duplicated_from: keturn/INED-datasette
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

metadata.json

@@ -0,0 +1,14 @@
+{
+  "title": "Imaginary Network Expanded Dataset",
+  "description": "Curated by Sygil",
+  "source_url": "https://github.com/Sygil-Dev/INE-dataset",
+  "databases": {
+    "INE": {
+      "tables": {
+        "images": {
+          "fts_table": "captions_fts"
+        }
+      }
+    }
+  }
+}

settings.json

@@ -0,0 +1,3 @@
+{
+  "default_page_size": 20
+}

src/import-git.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -x -e -o pipefail
+
+REPO="https://github.com/Sygil-Dev/INE-dataset.git"
+IMAGE_HOST="https://raw.githubusercontent.com/Sygil-Dev/INE-dataset/main/data/"
+
+# avoid cloning all the image files
+git clone --no-checkout --filter=blob:none --depth 1 "${REPO}" dataset
+
+# Beware `--no-cone` is deprecated, so this may stop working someday
+# https://git-scm.com/docs/git-sparse-checkout#_internalsnon_cone_problems
+git -C dataset sparse-checkout set --no-cone '/data/*.txt'
+git -C dataset checkout main
+
+python3 -m textdir2sql.loading dataset/data INE.db \
+    --image-host="${IMAGE_HOST}"

src/textdir2sql/loading.py

@@ -0,0 +1,90 @@
+import sqlite3
+from functools import partial
+from itertools import islice
+from pathlib import Path
+
+import click
+
+BATCH_SIZE=1024
+
+
+@click.command()
+@click.argument('input_dir', type=click.Path(exists=True, file_okay=False, path_type=Path))
+@click.argument('output', type=click.Path(dir_okay=False, writable=True, path_type=Path))
+@click.option('--image-host', help="base URL of images")
+@click.option('--explicit/--no-explicit', default=False)
+def main(input_dir: Path, output: Path, image_host: str, explicit:bool):
+    connection = sqlite3.connect(output)
+    try:
+        _main_with_connection(input_dir, connection, image_host, explicit)
+    finally:
+        connection.close()
+
+def _main_with_connection(input_dir: Path, connection: sqlite3.Connection, image_host: str=None, explicit=False):
+    connection.execute("CREATE TABLE IF NOT EXISTS "
+                       "    captions(image_key text PRIMARY KEY, caption text NOT NULL);")
+
+    if image_host:
+        connection.execute(f"""
+        CREATE VIEW IF NOT EXISTS images AS
+            SELECT {sql_quote(connection, image_host)} || image_key || '.jpg' AS image,
+            caption, 
+            rowid
+        FROM captions
+        """)
+
+    text_files = input_dir.glob("*.txt")
+
+    with click.progressbar(chunked(text_files, BATCH_SIZE)) as progress:
+        for batch in progress:
+            text_file: Path
+            pairs = ((text_file.stem, text_file.read_text())
+                for text_file in batch)
+            with connection:
+                connection.executemany("INSERT INTO captions(image_key, caption) "
+                                       "VALUES(?, ?) ", pairs)
+
+    if not explicit:
+        ratings = ["rating:unsafe", "rating:explicit", "rating:mature", "meta:nsfw",
+                   "subreddit:%nsfw"]
+        for rating in ratings:
+            with connection:
+                c = connection.execute("DELETE FROM captions WHERE caption LIKE ?",
+                                       (f"%{rating}%",))
+                print(f"Removed {c.rowcount} {rating} rows")
+
+    with connection:
+        # Add full-text search index
+        connection.execute("""CREATE VIRTUAL TABLE
+           captions_fts USING 
+           fts5(caption, image_key UNINDEXED, content=captions)
+        """)
+        connection.execute("""
+            INSERT INTO "captions_fts" (rowid, image_key, caption)
+            SELECT rowid, image_key, caption
+            FROM captions
+        """)
+
+
+def chunked(iterable, n):
+    return iter(partial(take, n, iter(iterable)), [])
+
+def take(n, iterable):
+    return list(islice(iterable, n))
+
+def sql_quote(connection, value: str) -> str:
+    """
+    Apply SQLite string quoting to a value, including wrapping it in single quotes.
+    :param value: String to quote
+    """
+    # Normally we would use .execute(sql, [params]) for escaping, but
+    # occasionally that isn't available - most notable when we need
+    # to include a "... DEFAULT 'value'" in a column definition.
+    return connection.execute(
+        # Use SQLite itself to correctly escape this string:
+        "SELECT quote(:value)",
+        {"value": value},
+    ).fetchone()[0]
+
+if __name__ == "__main__":
+    main()