Upload data/lilac.yml with huggingface_hub

data/lilac.yml

@@ -1,11 +1,14 @@
-# Lilac project config.
-# See https://lilacml.com/api_reference/index.html#lilac.Config for details.
-
 datasets:
   - namespace: lilac
     name: imdb
+    tags: []
     source:
       dataset_name: imdb
+      config_name: null
+      split: null
+      sample_size: null
+      revision: null
+      load_from_disk: false
       source_name: huggingface
     embeddings:
       - path: text
@@ -13,60 +16,70 @@ datasets:
     signals:
       - path: text
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: text
         signal:
           signal_name: pii
       - path: text
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
@@ -75,11 +88,18 @@ datasets:
       ui:
         media_paths:
           - text
+        markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
     name: open-asssistant-conversations
+    tags: []
     source:
       dataset_name: OpenAssistant/oasst1
+      config_name: null
+      split: null
+      sample_size: null
+      revision: null
+      load_from_disk: false
       source_name: huggingface
     embeddings:
       - path: text
@@ -87,66 +107,77 @@ datasets:
     signals:
       - path: text
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: text
         signal:
           signal_name: pii
       - path: text
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
@@ -155,12 +186,18 @@ datasets:
       ui:
         media_paths:
           - text
+        markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
     name: wikitext-2-raw-v1
+    tags: []
     source:
       dataset_name: wikitext
       config_name: wikitext-2-raw-v1
+      split: null
+      sample_size: null
+      revision: null
+      load_from_disk: false
       source_name: huggingface
     embeddings:
       - path: text
@@ -168,12 +205,14 @@ datasets:
     signals:
       - path: text
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: text
         signal:
           signal_name: pii
       - path: text
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: text
         signal:
@@ -183,58 +222,73 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
     settings:
       ui:
         media_paths:
           - text
+        markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
     name: squad_v2
+    tags: []
     source:
       dataset_name: squad_v2
+      config_name: null
+      split: null
+      sample_size: null
+      revision: null
+      load_from_disk: false
       source_name: huggingface
     embeddings:
       - path: context
@@ -242,72 +296,84 @@ datasets:
     signals:
       - path: context
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: context
         signal:
           signal_name: pii
       - path: context
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: context
         signal:
           signal_name: text_statistics
       - path: question
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: question
         signal:
           signal_name: pii
       - path: question
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: question
         signal:
@@ -317,6 +383,7 @@ datasets:
           - text
           - '*'
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path:
           - answers
@@ -329,6 +396,7 @@ datasets:
           - text
           - '*'
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path:
           - answers
@@ -341,48 +409,56 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path:
           - answers
@@ -392,6 +468,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path:
           - answers
@@ -401,6 +478,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - answers
@@ -410,6 +488,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path:
           - answers
@@ -419,6 +498,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - answers
@@ -428,6 +508,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path:
           - answers
@@ -437,6 +518,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path:
           - answers
@@ -446,6 +528,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path:
           - answers
@@ -455,6 +538,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
     settings:
       ui:
@@ -464,11 +548,18 @@ datasets:
           - - answers
             - text
             - '*'
+        markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
     name: databricks-dolly-15k-curated-en
+    tags: []
     source:
       dataset_name: argilla/databricks-dolly-15k-curated-en
+      config_name: null
+      split: null
+      sample_size: null
+      revision: null
+      load_from_disk: false
       source_name: huggingface
     embeddings:
       - path: original-context
@@ -481,36 +572,42 @@ datasets:
     signals:
       - path: original-instruction
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: original-instruction
         signal:
           signal_name: pii
       - path: original-instruction
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: original-instruction
         signal:
           signal_name: text_statistics
       - path: original-context
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: original-context
         signal:
           signal_name: pii
       - path: original-context
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: original-context
         signal:
           signal_name: text_statistics
       - path: original-response
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: original-response
         signal:
           signal_name: pii
       - path: original-response
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: original-response
         signal:
@@ -520,6 +617,7 @@ datasets:
           - value
           - '*'
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path:
           - new-instruction
@@ -532,6 +630,7 @@ datasets:
           - value
           - '*'
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path:
           - new-instruction
@@ -544,6 +643,7 @@ datasets:
           - value
           - '*'
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path:
           - new-context
@@ -556,6 +656,7 @@ datasets:
           - value
           - '*'
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path:
           - new-context
@@ -568,6 +669,7 @@ datasets:
           - value
           - '*'
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path:
           - new-response
@@ -580,6 +682,7 @@ datasets:
           - value
           - '*'
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path:
           - new-response
@@ -592,144 +695,168 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: original-instruction
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: original-instruction
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: original-instruction
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: original-instruction
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: original-instruction
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: original-instruction
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: original-instruction
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: original-context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: original-context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: original-context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: original-context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: original-context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: original-context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: original-context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: original-context
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: original-response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: original-response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: original-response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: original-response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: original-response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: original-response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: original-response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: original-response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path:
           - new-instruction
@@ -739,6 +866,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path:
           - new-instruction
@@ -748,6 +876,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - new-instruction
@@ -757,6 +886,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path:
           - new-instruction
@@ -766,6 +896,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - new-instruction
@@ -775,6 +906,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path:
           - new-instruction
@@ -784,6 +916,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path:
           - new-instruction
@@ -793,6 +926,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path:
           - new-instruction
@@ -802,6 +936,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path:
           - new-context
@@ -811,6 +946,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path:
           - new-context
@@ -820,6 +956,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - new-context
@@ -829,6 +966,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path:
           - new-context
@@ -838,6 +976,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - new-context
@@ -847,6 +986,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path:
           - new-context
@@ -856,6 +996,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path:
           - new-context
@@ -865,6 +1006,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path:
           - new-context
@@ -874,6 +1016,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path:
           - new-response
@@ -883,6 +1026,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path:
           - new-response
@@ -892,6 +1036,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - new-response
@@ -901,6 +1046,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path:
           - new-response
@@ -910,6 +1056,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - new-response
@@ -919,6 +1066,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path:
           - new-response
@@ -928,6 +1076,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path:
           - new-response
@@ -937,6 +1086,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path:
           - new-response
@@ -946,6 +1096,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
     settings:
       ui:
@@ -962,11 +1113,18 @@ datasets:
           - - new-response
             - value
             - '*'
+        markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
     name: piqa
+    tags: []
     source:
       dataset_name: piqa
+      config_name: null
+      split: null
+      sample_size: null
+      revision: null
+      load_from_disk: false
       source_name: huggingface
     embeddings:
       - path: goal
@@ -978,180 +1136,210 @@ datasets:
     signals:
       - path: goal
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: goal
         signal:
           signal_name: pii
       - path: goal
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: goal
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: goal
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: goal
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: goal
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: goal
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: goal
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: goal
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: goal
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: goal
         signal:
           signal_name: text_statistics
       - path: sol1
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: sol1
         signal:
           signal_name: pii
       - path: sol1
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: sol1
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: sol1
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: sol1
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: sol1
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: sol1
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: sol1
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: sol1
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: sol1
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: sol1
         signal:
           signal_name: text_statistics
       - path: sol2
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: sol2
         signal:
           signal_name: pii
       - path: sol2
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: sol2
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: sol2
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: sol2
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: sol2
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: sol2
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: sol2
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: sol2
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: sol2
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: sol2
         signal:
@@ -1162,12 +1350,18 @@ datasets:
           - sol1
           - sol2
           - goal
+        markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
     name: OpenOrca-100k
+    tags: []
     source:
       dataset_name: Open-Orca/OpenOrca
+      config_name: null
+      split: null
       sample_size: 100000
+      revision: null
+      load_from_disk: false
       source_name: huggingface
     embeddings:
       - path: question
@@ -1177,120 +1371,140 @@ datasets:
     signals:
       - path: question
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: question
         signal:
           signal_name: pii
       - path: question
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           signal_name: text_statistics
       - path: response
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: response
         signal:
           signal_name: pii
       - path: response
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: response
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: response
         signal:
@@ -1300,13 +1514,18 @@ datasets:
         media_paths:
           - question
           - response
+        markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
     name: opus100-en-es-validation
+    tags: []
     source:
       dataset_name: opus100
       config_name: en-es
       split: validation
+      sample_size: null
+      revision: null
+      load_from_disk: false
       source_name: huggingface
     embeddings:
       - path:
@@ -1322,6 +1541,7 @@ datasets:
           - translation
           - en
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path:
           - translation
@@ -1332,6 +1552,7 @@ datasets:
           - translation
           - en
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path:
           - translation
@@ -1340,6 +1561,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1348,6 +1570,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1356,6 +1579,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1364,6 +1588,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1372,6 +1597,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1380,6 +1606,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1388,6 +1615,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1396,6 +1624,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1406,6 +1635,7 @@ datasets:
           - translation
           - es
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path:
           - translation
@@ -1416,6 +1646,7 @@ datasets:
           - translation
           - es
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path:
           - translation
@@ -1424,6 +1655,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1432,6 +1664,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1440,6 +1673,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1448,6 +1682,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1456,6 +1691,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1464,6 +1700,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1472,6 +1709,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1480,6 +1718,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path:
           - translation
@@ -1493,12 +1732,18 @@ datasets:
             - es
           - - translation
             - en
+        markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
     name: mmlu_professional_law
+    tags: []
     source:
       dataset_name: cais/mmlu
       config_name: professional_law
+      split: null
+      sample_size: null
+      revision: null
+      load_from_disk: false
       source_name: huggingface
     embeddings:
       - path: question
@@ -1510,60 +1755,70 @@ datasets:
     signals:
       - path: question
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: question
         signal:
           signal_name: pii
       - path: question
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: question
         signal:
@@ -1572,6 +1827,7 @@ datasets:
           - choices
           - '*'
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path:
           - choices
@@ -1582,6 +1838,7 @@ datasets:
           - choices
           - '*'
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path:
           - choices
@@ -1590,6 +1847,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - choices
@@ -1598,6 +1856,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path:
           - choices
@@ -1606,6 +1865,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path:
           - choices
@@ -1614,6 +1874,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path:
           - choices
@@ -1622,6 +1883,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path:
           - choices
@@ -1630,6 +1892,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path:
           - choices
@@ -1638,6 +1901,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - choices
@@ -1646,6 +1910,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path:
           - choices
@@ -1654,6 +1919,7 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path:
           - choices
@@ -1666,12 +1932,18 @@ datasets:
           - question
           - - choices
             - '*'
+        markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
     name: pile-of-law-r-legaladvice
+    tags: []
     source:
       dataset_name: pile-of-law/pile-of-law
       config_name: r_legaladvice
+      split: null
+      sample_size: null
+      revision: null
+      load_from_disk: false
       source_name: huggingface
     embeddings:
       - path: text
@@ -1679,60 +1951,70 @@ datasets:
     signals:
       - path: text
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: text
         signal:
           signal_name: pii
       - path: text
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
@@ -1741,11 +2023,18 @@ datasets:
       ui:
         media_paths:
           - text
+        markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
     name: science-qa-derek-thomas
+    tags: []
     source:
       dataset_name: derek-thomas/ScienceQA
+      config_name: null
+      split: null
+      sample_size: null
+      revision: null
+      load_from_disk: false
       source_name: huggingface
     embeddings:
       - path: lecture
@@ -1753,12 +2042,14 @@ datasets:
     signals:
       - path: lecture
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: lecture
         signal:
           signal_name: pii
       - path: lecture
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: lecture
         signal:
@@ -1768,60 +2059,73 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: lecture
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: lecture
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: lecture
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: lecture
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: lecture
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: lecture
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: lecture
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
     settings:
       ui:
         media_paths:
           - lecture
+        markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
     name: enron-emails
+    tags: []
     source:
       dataset_name: EleutherAI/pile
       config_name: enron_emails
+      split: null
       sample_size: 100000
+      revision: null
+      load_from_disk: false
       source_name: huggingface
     embeddings:
       - path: text
@@ -1829,60 +2133,70 @@ datasets:
     signals:
       - path: text
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: text
         signal:
           signal_name: pii
       - path: text
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: text
         signal:
@@ -1891,12 +2205,17 @@ datasets:
       ui:
         media_paths:
           - text
+        markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
     name: the_movies_dataset
+    tags: []
     source:
       filepaths:
         - https://storage.googleapis.com/lilac-data/datasets/the_movies_dataset/the_movies_dataset.csv
+      delim: ','
+      header: true
+      names: null
       source_name: csv
     embeddings:
       - path: overview
@@ -1904,12 +2223,14 @@ datasets:
     signals:
       - path: overview
         signal:
+          threshold: 0.85
           signal_name: near_dup
       - path: overview
         signal:
           signal_name: pii
       - path: overview
         signal:
+          split_by_paragraph: false
           signal_name: lang_detection
       - path: overview
         signal:
@@ -1919,51 +2240,62 @@ datasets:
           embedding: gte-small
           namespace: lilac
           concept_name: legal-termination
+          draft: main
           signal_name: concept_score
       - path: overview
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: negative-sentiment
+          draft: main
           signal_name: concept_score
       - path: overview
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: non-english
+          draft: main
           signal_name: concept_score
       - path: overview
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: positive-sentiment
+          draft: main
           signal_name: concept_score
       - path: overview
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: profanity
+          draft: main
           signal_name: concept_score
       - path: overview
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: question
+          draft: main
           signal_name: concept_score
       - path: overview
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: source-code
+          draft: main
           signal_name: concept_score
       - path: overview
         signal:
           embedding: gte-small
           namespace: lilac
           concept_name: toxicity
+          draft: main
           signal_name: concept_score
     settings:
       ui:
         media_paths:
           - overview
+        markdown_paths: []
       preferred_embedding: gte-small
+signals: []
+concept_model_cache_embeddings: []