Push

.gitattributes

@@ -1,106 +0,0 @@
-data/.cache/lilac/concept/lilac/profanity/gte-small.pkl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/concept/lilac/toxicity/gte-small.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/goal/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/goal/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/goal/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol1/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol1/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol1/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol2/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol2/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol2/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/science-qa-derek-thomas/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/science-qa-derek-thomas/lecture/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/science-qa-derek-thomas/lecture/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/science-qa-derek-thomas/lecture/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/enron-emails/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/enron-emails/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/enron-emails/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/enron-emails/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/enron-emails/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/enron-emails/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/enron-emails/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/databricks-dolly-15k-curated-en/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/databricks-dolly-15k-curated-en/new-context/value/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/databricks-dolly-15k-curated-en/original-context/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/question/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/question/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/question/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/question/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/question/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/question/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/question/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/response/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/response/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/response/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/response/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/response/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/response/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/OpenOrca-100k/response/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/opus100-en-us-validation/translation/en/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/opus100-en-us-validation/translation/es/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -14,6 +14,9 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 COPY .env .
 COPY .env.demo .
+COPY demo_config.yml .
+# Copy the README so we can read the datasets from the HuggingFace config.
+COPY README.md .
 COPY LICENSE .
 
 # Copy python files.

README.md

@@ -1,8 +1,13 @@
 ---
-title: Lilac
-emoji: 🌷
+app_port: 5432
 colorFrom: purple
 colorTo: purple
+datasets: [lilacai/lilac-piqa, lilacai/lilac-science-qa-derek-thomas, lilacai/lilac-enron-emails,
+  lilacai/lilac-mmlu_professional_law, lilacai/lilac-pile-of-law-r-legaladvice, lilacai/lilac-open-asssistant-conversations,
+  lilacai/lilac-squad_v2, lilacai/lilac-imdb, lilacai/lilac-databricks-dolly-15k-curated-en,
+  lilacai/lilac-OpenOrca-100k, lilacai/lilac-wikitext-2-raw-v1, lilacai/lilac-opus100-en-us-validation]
+emoji: "\U0001F337"
 sdk: docker
-app_port: 5432
+title: Lilac
+
 ---

data/.cache/lilac/concept/lilac/profanity/gte-small.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7cf7bf81552f4965f217d537747e806715f508250c2095793743ef276ecddb18
-size 1672960

data/.cache/lilac/concept/lilac/toxicity/gte-small.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:097d149cbb575e054ea00eac3bbae56498dcd4b0e9ef7b9d32231efc538acb89
-size 1886446

data/datasets/lilac/OpenOrca-100k/config.yml

@@ -1,28 +0,0 @@
-embeddings:
-- {embedding: gte-small, path: response}
-- {embedding: gte-small, path: question}
-name: OpenOrca-100k
-namespace: local
-settings:
-  preferred_embedding: gte-small
-  ui:
-    media_paths: [question, response]
-signals:
-- path: question
-  signal: {signal_name: near_dup}
-- path: question
-  signal: {signal_name: text_statistics}
-- path: question
-  signal: {signal_name: pii}
-- path: question
-  signal: {signal_name: lang_detection}
-- path: response
-  signal: {signal_name: near_dup}
-- path: response
-  signal: {signal_name: text_statistics}
-- path: response
-  signal: {signal_name: pii}
-- path: response
-  signal: {signal_name: lang_detection}
-source: {dataset_name: Open-Orca/OpenOrca, sample_size: 100000, source_name: huggingface}
-tags: [machine-learning]

data/datasets/lilac/OpenOrca-100k/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f95588367446af55ccc2f089092779670c57308ee1f72a849e41f22e126d5052
-size 105147761

data/datasets/lilac/OpenOrca-100k/manifest.json

@@ -1,24 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "data_schema": {
-    "fields": {
-      "id": {
-        "dtype": "string"
-      },
-      "system_prompt": {
-        "dtype": "string"
-      },
-      "question": {
-        "dtype": "string"
-      },
-      "response": {
-        "dtype": "string"
-      },
-      "__hfsplit__": {
-        "dtype": "string"
-      }
-    }
-  }
-}

data/datasets/lilac/OpenOrca-100k/question/gte-small/hnsw.hnswlib.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8f154c2dc5a0d69538c39df10508fe05cc36fb5489b61c303c9869320ef04581
-size 596704812

data/datasets/lilac/OpenOrca-100k/question/gte-small/hnsw.lookup.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3c0bac1790aa5247eb288c2a828a92eb313090b36a015665f6aae42e5a4dcb18
-size 9378299

data/datasets/lilac/OpenOrca-100k/question/gte-small/signal_manifest.json

@@ -1,32 +0,0 @@
-{
-  "files": [],
-  "parquet_id": "question.gte-small",
-  "data_schema": {
-    "fields": {
-      "question": {
-        "fields": {
-          "gte-small": {
-            "repeated_field": {
-              "fields": {
-                "embedding": {
-                  "dtype": "embedding"
-                }
-              },
-              "dtype": "string_span"
-            },
-            "signal": {
-              "signal_name": "gte-small"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "gte-small"
-  },
-  "enriched_path": [
-    "question"
-  ],
-  "vector_store": "hnsw"
-}

data/datasets/lilac/OpenOrca-100k/question/gte-small/spans.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ef19c506d4af4eab34aec3b280663687002db0792108b84d313f8ab6f532aa6c
-size 6922769

data/datasets/lilac/OpenOrca-100k/question/lang_detection/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b1a09e7085a4019205c62c28e6dcb46254fea37243e8087346d9c7298e05f9e1
-size 3327888

data/datasets/lilac/OpenOrca-100k/question/lang_detection/signal_manifest.json

@@ -1,28 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "question.lang_detection",
-  "data_schema": {
-    "fields": {
-      "question": {
-        "fields": {
-          "lang_detection": {
-            "dtype": "string",
-            "signal": {
-              "split_by_paragraph": false,
-              "signal_name": "lang_detection"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "split_by_paragraph": false,
-    "signal_name": "lang_detection"
-  },
-  "enriched_path": [
-    "question"
-  ]
-}

data/datasets/lilac/OpenOrca-100k/question/near_dup/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e1ff51f57fb136ab846d0c34a248aca4ef86d09fa0945737cd2c276d2f5dcb7d
-size 3884385

data/datasets/lilac/OpenOrca-100k/question/near_dup/signal_manifest.json

@@ -1,33 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "question.near_dup",
-  "data_schema": {
-    "fields": {
-      "question": {
-        "fields": {
-          "near_dup": {
-            "fields": {
-              "cluster_id": {
-                "dtype": "uint32",
-                "categorical": true
-              }
-            },
-            "signal": {
-              "threshold": 0.85,
-              "signal_name": "near_dup"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "threshold": 0.85,
-    "signal_name": "near_dup"
-  },
-  "enriched_path": [
-    "question"
-  ]
-}

data/datasets/lilac/OpenOrca-100k/question/pii/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:acc4cd2ae7c51b4450d159c63fee3e9739b3c1d5a36cfbf3bf45fe29e2ac15b5
-size 3317869

data/datasets/lilac/OpenOrca-100k/question/pii/signal_manifest.json

@@ -1,42 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "question.pii",
-  "data_schema": {
-    "fields": {
-      "question": {
-        "fields": {
-          "pii": {
-            "fields": {
-              "emails": {
-                "repeated_field": {
-                  "dtype": "string_span"
-                }
-              },
-              "ip_addresses": {
-                "repeated_field": {
-                  "dtype": "string_span"
-                }
-              },
-              "secrets": {
-                "repeated_field": {
-                  "dtype": "string_span"
-                }
-              }
-            },
-            "signal": {
-              "signal_name": "pii"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "pii"
-  },
-  "enriched_path": [
-    "question"
-  ]
-}

data/datasets/lilac/OpenOrca-100k/question/text_statistics/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b6703f93314760ee15d64532812a601c85d2f411254c1d809c6b3f558cc1c7c7
-size 4321496

data/datasets/lilac/OpenOrca-100k/question/text_statistics/signal_manifest.json

@@ -1,56 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "question.text_statistics",
-  "data_schema": {
-    "fields": {
-      "question": {
-        "fields": {
-          "text_statistics": {
-            "fields": {
-              "num_characters": {
-                "dtype": "int32"
-              },
-              "readability": {
-                "dtype": "float32"
-              },
-              "log(type_token_ratio)": {
-                "dtype": "float32"
-              },
-              "frac_non_ascii": {
-                "dtype": "float32",
-                "bins": [
-                  [
-                    "Low",
-                    null,
-                    0.15
-                  ],
-                  [
-                    "Medium",
-                    0.15,
-                    0.3
-                  ],
-                  [
-                    "High",
-                    0.3,
-                    null
-                  ]
-                ]
-              }
-            },
-            "signal": {
-              "signal_name": "text_statistics"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "text_statistics"
-  },
-  "enriched_path": [
-    "question"
-  ]
-}

data/datasets/lilac/OpenOrca-100k/response/gte-small/hnsw.hnswlib.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c2374770842450f7d1712e2d56bc2e50bb1579af4cda061df2baf4631965dbcd
-size 482647596

data/datasets/lilac/OpenOrca-100k/response/gte-small/hnsw.lookup.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:afd6636965df6ed8f6aadd52a9638edf201c36dd470b816e6488e5417dcfe3c4
-size 8159214

data/datasets/lilac/OpenOrca-100k/response/gte-small/signal_manifest.json

@@ -1,32 +0,0 @@
-{
-  "files": [],
-  "parquet_id": "response.gte-small",
-  "data_schema": {
-    "fields": {
-      "response": {
-        "fields": {
-          "gte-small": {
-            "repeated_field": {
-              "fields": {
-                "embedding": {
-                  "dtype": "embedding"
-                }
-              },
-              "dtype": "string_span"
-            },
-            "signal": {
-              "signal_name": "gte-small"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "gte-small"
-  },
-  "enriched_path": [
-    "response"
-  ],
-  "vector_store": "hnsw"
-}

data/datasets/lilac/OpenOrca-100k/response/gte-small/spans.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cf61917d291a1e3157ca017b4eacdf8983bf8094b3b22d710031381927f19b16
-size 6373377

data/datasets/lilac/OpenOrca-100k/response/lang_detection/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:62af2b56e9bf3cbcddbceab6f858fc35fef50953b73b06a7da3bc1d2e62d3a53
-size 3339983

data/datasets/lilac/OpenOrca-100k/response/lang_detection/signal_manifest.json

@@ -1,28 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "response.lang_detection",
-  "data_schema": {
-    "fields": {
-      "response": {
-        "fields": {
-          "lang_detection": {
-            "dtype": "string",
-            "signal": {
-              "split_by_paragraph": false,
-              "signal_name": "lang_detection"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "split_by_paragraph": false,
-    "signal_name": "lang_detection"
-  },
-  "enriched_path": [
-    "response"
-  ]
-}

data/datasets/lilac/OpenOrca-100k/response/near_dup/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cc30679d1a2e6f2d3d45b2f145932daebf8a3f6ae4b73cfa9da3dbf5c495967d
-size 3902985

data/datasets/lilac/OpenOrca-100k/response/near_dup/signal_manifest.json

@@ -1,33 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "response.near_dup",
-  "data_schema": {
-    "fields": {
-      "response": {
-        "fields": {
-          "near_dup": {
-            "fields": {
-              "cluster_id": {
-                "dtype": "uint32",
-                "categorical": true
-              }
-            },
-            "signal": {
-              "threshold": 0.85,
-              "signal_name": "near_dup"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "threshold": 0.85,
-    "signal_name": "near_dup"
-  },
-  "enriched_path": [
-    "response"
-  ]
-}

data/datasets/lilac/OpenOrca-100k/response/pii/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f3b5830b33a8ddbe000b1b4403ef882731243075acc6416b5f673c90d4bf25ac
-size 3313965

data/datasets/lilac/OpenOrca-100k/response/pii/signal_manifest.json

@@ -1,42 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "response.pii",
-  "data_schema": {
-    "fields": {
-      "response": {
-        "fields": {
-          "pii": {
-            "fields": {
-              "emails": {
-                "repeated_field": {
-                  "dtype": "string_span"
-                }
-              },
-              "ip_addresses": {
-                "repeated_field": {
-                  "dtype": "string_span"
-                }
-              },
-              "secrets": {
-                "repeated_field": {
-                  "dtype": "string_span"
-                }
-              }
-            },
-            "signal": {
-              "signal_name": "pii"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "pii"
-  },
-  "enriched_path": [
-    "response"
-  ]
-}

data/datasets/lilac/OpenOrca-100k/response/text_statistics/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0e5bae031d37b7df9a3df49a616d58a8f9962307750039c1736b8faa56d8501a
-size 4281305

data/datasets/lilac/OpenOrca-100k/response/text_statistics/signal_manifest.json

@@ -1,56 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "response.text_statistics",
-  "data_schema": {
-    "fields": {
-      "response": {
-        "fields": {
-          "text_statistics": {
-            "fields": {
-              "num_characters": {
-                "dtype": "int32"
-              },
-              "readability": {
-                "dtype": "float32"
-              },
-              "log(type_token_ratio)": {
-                "dtype": "float32"
-              },
-              "frac_non_ascii": {
-                "dtype": "float32",
-                "bins": [
-                  [
-                    "Low",
-                    null,
-                    0.15
-                  ],
-                  [
-                    "Medium",
-                    0.15,
-                    0.3
-                  ],
-                  [
-                    "High",
-                    0.3,
-                    null
-                  ]
-                ]
-              }
-            },
-            "signal": {
-              "signal_name": "text_statistics"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "text_statistics"
-  },
-  "enriched_path": [
-    "response"
-  ]
-}

data/datasets/lilac/databricks-dolly-15k-curated-en/config.yml

@@ -1,67 +0,0 @@
-embeddings:
-- embedding: gte-small
-  path: [new-context, value, '*']
-- {embedding: gte-small, path: original-context}
-name: databricks-dolly-15k-curated-en
-namespace: lilac
-settings:
-  preferred_embedding: gte-small
-  ui:
-    media_paths:
-    - original-instruction
-    - original-context
-    - original-response
-    - [new-instruction, value, '*']
-    - [new-context, value, '*']
-    - [new-response, value, '*']
-signals:
-- path: original-instruction
-  signal: {signal_name: near_dup}
-- path: original-instruction
-  signal: {signal_name: text_statistics}
-- path: original-instruction
-  signal: {signal_name: pii}
-- path: original-instruction
-  signal: {signal_name: lang_detection}
-- path: original-context
-  signal: {signal_name: near_dup}
-- path: original-context
-  signal: {signal_name: text_statistics}
-- path: original-context
-  signal: {signal_name: lang_detection}
-- path: original-context
-  signal: {signal_name: pii}
-- path: original-response
-  signal: {signal_name: near_dup}
-- path: original-response
-  signal: {signal_name: text_statistics}
-- path: original-response
-  signal: {signal_name: pii}
-- path: original-response
-  signal: {signal_name: lang_detection}
-- path: [new-instruction, value, '*']
-  signal: {signal_name: near_dup}
-- path: [new-instruction, value, '*']
-  signal: {signal_name: text_statistics}
-- path: [new-instruction, value, '*']
-  signal: {signal_name: pii}
-- path: [new-instruction, value, '*']
-  signal: {signal_name: lang_detection}
-- path: [new-context, value, '*']
-  signal: {signal_name: near_dup}
-- path: [new-context, value, '*']
-  signal: {signal_name: text_statistics}
-- path: [new-context, value, '*']
-  signal: {signal_name: lang_detection}
-- path: [new-context, value, '*']
-  signal: {signal_name: pii}
-- path: [new-response, value, '*']
-  signal: {signal_name: near_dup}
-- path: [new-response, value, '*']
-  signal: {signal_name: text_statistics}
-- path: [new-response, value, '*']
-  signal: {signal_name: pii}
-- path: [new-response, value, '*']
-  signal: {signal_name: lang_detection}
-source: {dataset_name: argilla/databricks-dolly-15k-curated-en, source_name: huggingface}
-tags: [machine-learning]

data/datasets/lilac/databricks-dolly-15k-curated-en/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ad225b50d5880a097ea66eb4ca70fc529c0321cf8a5652bd8fbe7a638d016851
-size 15882489

data/datasets/lilac/databricks-dolly-15k-curated-en/manifest.json

@@ -1,87 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "data_schema": {
-    "fields": {
-      "id": {
-        "dtype": "string"
-      },
-      "category": {
-        "dtype": "string"
-      },
-      "original-instruction": {
-        "dtype": "string"
-      },
-      "original-context": {
-        "dtype": "string"
-      },
-      "original-response": {
-        "dtype": "string"
-      },
-      "new-instruction": {
-        "fields": {
-          "user_id": {
-            "repeated_field": {
-              "dtype": "string"
-            }
-          },
-          "value": {
-            "repeated_field": {
-              "dtype": "string"
-            }
-          },
-          "status": {
-            "repeated_field": {
-              "dtype": "string"
-            }
-          }
-        }
-      },
-      "new-context": {
-        "fields": {
-          "user_id": {
-            "repeated_field": {
-              "dtype": "string"
-            }
-          },
-          "value": {
-            "repeated_field": {
-              "dtype": "string"
-            }
-          },
-          "status": {
-            "repeated_field": {
-              "dtype": "string"
-            }
-          }
-        }
-      },
-      "new-response": {
-        "fields": {
-          "user_id": {
-            "repeated_field": {
-              "dtype": "string"
-            }
-          },
-          "value": {
-            "repeated_field": {
-              "dtype": "string"
-            }
-          },
-          "status": {
-            "repeated_field": {
-              "dtype": "string"
-            }
-          }
-        }
-      },
-      "external_id": {
-        "dtype": "string"
-      },
-      "__hfsplit__": {
-        "dtype": "string"
-      }
-    }
-  }
-}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-context/value/gte-small/hnsw.hnswlib.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c879460250e68b6195eed6b48afa2fa2a7b8127483a299818a13f82ed7fea8dc
-size 32553584

data/datasets/lilac/databricks-dolly-15k-curated-en/new-context/value/gte-small/signal_manifest.json

@@ -1,40 +0,0 @@
-{
-  "files": [],
-  "parquet_id": "new-context.value.gte-small",
-  "data_schema": {
-    "fields": {
-      "new-context": {
-        "fields": {
-          "value": {
-            "repeated_field": {
-              "fields": {
-                "gte-small": {
-                  "repeated_field": {
-                    "fields": {
-                      "embedding": {
-                        "dtype": "embedding"
-                      }
-                    },
-                    "dtype": "string_span"
-                  },
-                  "signal": {
-                    "signal_name": "gte-small"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "gte-small"
-  },
-  "enriched_path": [
-    "new-context",
-    "value",
-    "*"
-  ],
-  "vector_store": "hnsw"
-}