Push to HF space

Dockerfile

@@ -1,9 +1,12 @@
-# NOTE: When we upgrade to 3.11 we can use a slimmer docker image which comes with gcc.
-FROM python:3.9-bullseye
+FROM python:3.11-slim-bullseye
 
 # Allow statements and log messages to immediately appear in the Knative logs
 ENV PYTHONUNBUFFERED True
 
+# Adds GCC and other build tools so we can compile hnswlib and other native/C++ deps.
+RUN apt-get update --fix-missing && apt-get install -y --fix-missing build-essential && \
+  rm -rf /var/lib/apt/lists/*
+
 # See: https://huggingface.co/docs/hub/spaces-sdks-docker#permissions
 RUN useradd -m -u 1000 user
 USER user
@@ -39,4 +42,5 @@ COPY --chown=user docker_start.sh ./
 RUN mkdir -p ./data
 RUN chown -R user ./data
 
+EXPOSE 8000
 CMD ["bash", "docker_start.sh"]

data/lilac.yml

@@ -230,10 +230,13 @@ datasets:
           signal_name: concept_score
       - path: text
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
+      - path: text
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
     settings:
       ui:
         media_paths:
@@ -407,10 +410,13 @@ datasets:
           signal_name: concept_score
       - path: text
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
+      - path: text
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
     settings:
       ui:
         media_paths:
@@ -576,10 +582,13 @@ datasets:
           signal_name: concept_score
       - path: text
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
+      - path: text
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
     settings:
       ui:
         media_paths:
@@ -605,6 +614,20 @@ datasets:
           - value
           - '*'
         embedding: gte-small
+      - path: original-instruction
+        embedding: gte-small
+      - path: original-response
+        embedding: gte-small
+      - path:
+          - new-instruction
+          - value
+          - '*'
+        embedding: gte-small
+      - path:
+          - new-response
+          - value
+          - '*'
+        embedding: gte-small
     signals:
       - path: original-instruction
         signal:
@@ -2856,9 +2879,6 @@ datasets:
           signal_name: concept_score
       - path: original-instruction
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
       - path: original-instruction
         signal:
@@ -2926,15 +2946,9 @@ datasets:
           signal_name: concept_score
       - path: original-context
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
       - path: original-response
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
       - path: original-response
         signal:
@@ -3000,15 +3014,355 @@ datasets:
           version: null
           draft: main
           signal_name: concept_score
+      - path:
+          - new-instruction
+          - value
+          - '*'
+        signal:
+          signal_name: cluster_dbscan
+      - path:
+          - new-instruction
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: legal-termination
+          version: null
+          draft: main
+          signal_name: concept_score
       - path:
           - new-instruction
           - value
           - '*'
         signal:
           embedding: gte-small
-          eps: 0.05
-          min_samples: 5
+          namespace: lilac
+          concept_name: negative-sentiment
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-instruction
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: non-english
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-instruction
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: positive-sentiment
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-instruction
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: profanity
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-instruction
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: question
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-instruction
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: source-code
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-instruction
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: toxicity
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-context
+          - value
+          - '*'
+        signal:
           signal_name: cluster_dbscan
+      - path:
+          - new-response
+          - value
+          - '*'
+        signal:
+          signal_name: cluster_dbscan
+      - path:
+          - new-response
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: legal-termination
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-response
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: negative-sentiment
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-response
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: non-english
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-response
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: positive-sentiment
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-response
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: profanity
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-response
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: question
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-response
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: source-code
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-response
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: toxicity
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-instruction
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
+      - path: original-instruction
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: legal-termination
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-instruction
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: negative-sentiment
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-instruction
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: non-english
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-instruction
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: positive-sentiment
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-instruction
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: profanity
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-instruction
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: question
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-instruction
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: source-code
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-instruction
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: toxicity
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-context
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
+      - path: original-response
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
+      - path: original-response
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: legal-termination
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-response
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: negative-sentiment
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-response
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: non-english
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-response
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: positive-sentiment
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-response
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: profanity
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-response
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: question
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-response
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: source-code
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path: original-response
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: toxicity
+          version: null
+          draft: main
+          signal_name: concept_score
+      - path:
+          - new-instruction
+          - value
+          - '*'
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
       - path:
           - new-instruction
           - value
@@ -3103,18 +3457,18 @@ datasets:
           - '*'
         signal:
           embedding: gte-small
-          eps: 0.05
-          min_samples: 5
-          signal_name: cluster_dbscan
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
       - path:
           - new-response
           - value
           - '*'
         signal:
           embedding: gte-small
-          eps: 0.05
-          min_samples: 5
-          signal_name: cluster_dbscan
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
       - path:
           - new-response
           - value
@@ -3784,16 +4138,22 @@ datasets:
           signal_name: concept_score
       - path: question
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
       - path: response
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
+      - path: question
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
+      - path: response
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
     settings:
       ui:
         media_paths:
@@ -4190,18 +4550,28 @@ datasets:
           - translation
           - es
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
       - path:
           - translation
           - en
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
+      - path:
+          - translation
+          - es
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
+      - path:
+          - translation
+          - en
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
     settings:
       ui:
         media_paths:
@@ -4370,10 +4740,13 @@ datasets:
           signal_name: concept_score
       - path: lecture
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
+      - path: lecture
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
     settings:
       ui:
         media_paths:
@@ -4539,10 +4912,13 @@ datasets:
           signal_name: concept_score
       - path: text
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
+      - path: text
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
     settings:
       ui:
         media_paths:
@@ -4707,10 +5083,13 @@ datasets:
           signal_name: concept_score
       - path: overview
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
+      - path: overview
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
     settings:
       ui:
         media_paths:
@@ -5026,24 +5405,37 @@ datasets:
           - outline
           - '*'
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
       - path:
           - concepts
           - '*'
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
       - path: markdown
         signal:
-          embedding: gte-small
-          eps: 0.05
-          min_samples: 5
           signal_name: cluster_dbscan
+      - path:
+          - outline
+          - '*'
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
+      - path:
+          - concepts
+          - '*'
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
+      - path: markdown
+        signal:
+          embedding: gte-small
+          min_cluster_size: 5
+          umap_n_components: 10
+          signal_name: cluster_hdbscan
     settings:
       ui:
         media_paths:

docker_start.sh

@@ -5,6 +5,6 @@ set -e
 
 lilac hf-docker-start
 gunicorn lilac.server:app \
-  --bind 0.0.0.0:5432 \
+  --bind 0.0.0.0:8000 \
   --preload -k uvicorn.workers.UvicornWorker \
   --timeout 120