Upload folder using huggingface_hub

.gitattributes

@@ -76,3 +76,6 @@ data/datasets/lilac/imdb/text/lang_detection/data-00000-of-00001.parquet filter=
 data/datasets/lilac/imdb/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/imdb/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/imdb/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/databricks-dolly-15k-curated-en/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/databricks-dolly-15k-curated-en/new-context/value/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/databricks-dolly-15k-curated-en/original-context/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text

data/datasets/lilac/databricks-dolly-15k-curated-en/config.yml

@@ -0,0 +1,67 @@
+embeddings:
+- embedding: gte-small
+  path: [new-context, value, '*']
+- {embedding: gte-small, path: original-context}
+name: databricks-dolly-15k-curated-en
+namespace: lilac
+settings:
+  preferred_embedding: gte-small
+  ui:
+    media_paths:
+    - original-instruction
+    - original-context
+    - original-response
+    - [new-instruction, value, '*']
+    - [new-context, value, '*']
+    - [new-response, value, '*']
+signals:
+- path: original-instruction
+  signal: {signal_name: near_dup}
+- path: original-instruction
+  signal: {signal_name: text_statistics}
+- path: original-instruction
+  signal: {signal_name: pii}
+- path: original-instruction
+  signal: {signal_name: lang_detection}
+- path: original-context
+  signal: {signal_name: near_dup}
+- path: original-context
+  signal: {signal_name: text_statistics}
+- path: original-context
+  signal: {signal_name: lang_detection}
+- path: original-context
+  signal: {signal_name: pii}
+- path: original-response
+  signal: {signal_name: near_dup}
+- path: original-response
+  signal: {signal_name: text_statistics}
+- path: original-response
+  signal: {signal_name: pii}
+- path: original-response
+  signal: {signal_name: lang_detection}
+- path: [new-instruction, value, '*']
+  signal: {signal_name: near_dup}
+- path: [new-instruction, value, '*']
+  signal: {signal_name: text_statistics}
+- path: [new-instruction, value, '*']
+  signal: {signal_name: pii}
+- path: [new-instruction, value, '*']
+  signal: {signal_name: lang_detection}
+- path: [new-context, value, '*']
+  signal: {signal_name: near_dup}
+- path: [new-context, value, '*']
+  signal: {signal_name: text_statistics}
+- path: [new-context, value, '*']
+  signal: {signal_name: lang_detection}
+- path: [new-context, value, '*']
+  signal: {signal_name: pii}
+- path: [new-response, value, '*']
+  signal: {signal_name: near_dup}
+- path: [new-response, value, '*']
+  signal: {signal_name: text_statistics}
+- path: [new-response, value, '*']
+  signal: {signal_name: pii}
+- path: [new-response, value, '*']
+  signal: {signal_name: lang_detection}
+source: {dataset_name: argilla/databricks-dolly-15k-curated-en, source_name: huggingface}
+tags: [machine-learning]

data/datasets/lilac/databricks-dolly-15k-curated-en/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad225b50d5880a097ea66eb4ca70fc529c0321cf8a5652bd8fbe7a638d016851
+size 15882489

data/datasets/lilac/databricks-dolly-15k-curated-en/manifest.json

@@ -0,0 +1,87 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "data_schema": {
+    "fields": {
+      "id": {
+        "dtype": "string"
+      },
+      "category": {
+        "dtype": "string"
+      },
+      "original-instruction": {
+        "dtype": "string"
+      },
+      "original-context": {
+        "dtype": "string"
+      },
+      "original-response": {
+        "dtype": "string"
+      },
+      "new-instruction": {
+        "fields": {
+          "user_id": {
+            "repeated_field": {
+              "dtype": "string"
+            }
+          },
+          "value": {
+            "repeated_field": {
+              "dtype": "string"
+            }
+          },
+          "status": {
+            "repeated_field": {
+              "dtype": "string"
+            }
+          }
+        }
+      },
+      "new-context": {
+        "fields": {
+          "user_id": {
+            "repeated_field": {
+              "dtype": "string"
+            }
+          },
+          "value": {
+            "repeated_field": {
+              "dtype": "string"
+            }
+          },
+          "status": {
+            "repeated_field": {
+              "dtype": "string"
+            }
+          }
+        }
+      },
+      "new-response": {
+        "fields": {
+          "user_id": {
+            "repeated_field": {
+              "dtype": "string"
+            }
+          },
+          "value": {
+            "repeated_field": {
+              "dtype": "string"
+            }
+          },
+          "status": {
+            "repeated_field": {
+              "dtype": "string"
+            }
+          }
+        }
+      },
+      "external_id": {
+        "dtype": "string"
+      },
+      "__hfsplit__": {
+        "dtype": "string"
+      }
+    }
+  }
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-context/value/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c879460250e68b6195eed6b48afa2fa2a7b8127483a299818a13f82ed7fea8dc
+size 32553584

data/datasets/lilac/databricks-dolly-15k-curated-en/new-context/value/gte-small/signal_manifest.json

@@ -0,0 +1,40 @@
+{
+  "files": [],
+  "parquet_id": "new-context.value.gte-small",
+  "data_schema": {
+    "fields": {
+      "new-context": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "gte-small": {
+                  "repeated_field": {
+                    "fields": {
+                      "embedding": {
+                        "dtype": "embedding"
+                      }
+                    },
+                    "dtype": "string_span"
+                  },
+                  "signal": {
+                    "signal_name": "gte-small"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "new-context",
+    "value",
+    "*"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-context/value/lang_detection/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "new-context.value.lang_detection",
+  "data_schema": {
+    "fields": {
+      "new-context": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "lang_detection": {
+                  "dtype": "string",
+                  "signal": {
+                    "split_by_paragraph": false,
+                    "signal_name": "lang_detection"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "new-context",
+    "value",
+    "*"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-context/value/near_dup/signal_manifest.json

@@ -0,0 +1,41 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "new-context.value.near_dup",
+  "data_schema": {
+    "fields": {
+      "new-context": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "near_dup": {
+                  "fields": {
+                    "cluster_id": {
+                      "dtype": "uint32",
+                      "categorical": true
+                    }
+                  },
+                  "signal": {
+                    "threshold": 0.85,
+                    "signal_name": "near_dup"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "new-context",
+    "value",
+    "*"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-context/value/pii/signal_manifest.json

@@ -0,0 +1,50 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "new-context.value.pii",
+  "data_schema": {
+    "fields": {
+      "new-context": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "pii": {
+                  "fields": {
+                    "emails": {
+                      "repeated_field": {
+                        "dtype": "string_span"
+                      }
+                    },
+                    "ip_addresses": {
+                      "repeated_field": {
+                        "dtype": "string_span"
+                      }
+                    },
+                    "secrets": {
+                      "repeated_field": {
+                        "dtype": "string_span"
+                      }
+                    }
+                  },
+                  "signal": {
+                    "signal_name": "pii"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "new-context",
+    "value",
+    "*"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-context/value/text_statistics/signal_manifest.json

@@ -0,0 +1,64 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "new-context.value.text_statistics",
+  "data_schema": {
+    "fields": {
+      "new-context": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "text_statistics": {
+                  "fields": {
+                    "num_characters": {
+                      "dtype": "int32"
+                    },
+                    "readability": {
+                      "dtype": "float32"
+                    },
+                    "log(type_token_ratio)": {
+                      "dtype": "float32"
+                    },
+                    "frac_non_ascii": {
+                      "dtype": "float32",
+                      "bins": [
+                        [
+                          "Low",
+                          null,
+                          0.15
+                        ],
+                        [
+                          "Medium",
+                          0.15,
+                          0.3
+                        ],
+                        [
+                          "High",
+                          0.3,
+                          null
+                        ]
+                      ]
+                    }
+                  },
+                  "signal": {
+                    "signal_name": "text_statistics"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "new-context",
+    "value",
+    "*"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-instruction/value/lang_detection/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "new-instruction.value.lang_detection",
+  "data_schema": {
+    "fields": {
+      "new-instruction": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "lang_detection": {
+                  "dtype": "string",
+                  "signal": {
+                    "split_by_paragraph": false,
+                    "signal_name": "lang_detection"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "new-instruction",
+    "value",
+    "*"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-instruction/value/near_dup/signal_manifest.json

@@ -0,0 +1,41 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "new-instruction.value.near_dup",
+  "data_schema": {
+    "fields": {
+      "new-instruction": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "near_dup": {
+                  "fields": {
+                    "cluster_id": {
+                      "dtype": "uint32",
+                      "categorical": true
+                    }
+                  },
+                  "signal": {
+                    "threshold": 0.85,
+                    "signal_name": "near_dup"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "new-instruction",
+    "value",
+    "*"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-instruction/value/pii/signal_manifest.json

@@ -0,0 +1,50 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "new-instruction.value.pii",
+  "data_schema": {
+    "fields": {
+      "new-instruction": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "pii": {
+                  "fields": {
+                    "emails": {
+                      "repeated_field": {
+                        "dtype": "string_span"
+                      }
+                    },
+                    "ip_addresses": {
+                      "repeated_field": {
+                        "dtype": "string_span"
+                      }
+                    },
+                    "secrets": {
+                      "repeated_field": {
+                        "dtype": "string_span"
+                      }
+                    }
+                  },
+                  "signal": {
+                    "signal_name": "pii"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "new-instruction",
+    "value",
+    "*"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-instruction/value/text_statistics/signal_manifest.json

@@ -0,0 +1,64 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "new-instruction.value.text_statistics",
+  "data_schema": {
+    "fields": {
+      "new-instruction": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "text_statistics": {
+                  "fields": {
+                    "num_characters": {
+                      "dtype": "int32"
+                    },
+                    "readability": {
+                      "dtype": "float32"
+                    },
+                    "log(type_token_ratio)": {
+                      "dtype": "float32"
+                    },
+                    "frac_non_ascii": {
+                      "dtype": "float32",
+                      "bins": [
+                        [
+                          "Low",
+                          null,
+                          0.15
+                        ],
+                        [
+                          "Medium",
+                          0.15,
+                          0.3
+                        ],
+                        [
+                          "High",
+                          0.3,
+                          null
+                        ]
+                      ]
+                    }
+                  },
+                  "signal": {
+                    "signal_name": "text_statistics"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "new-instruction",
+    "value",
+    "*"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-response/value/lang_detection/signal_manifest.json

@@ -0,0 +1,36 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "new-response.value.lang_detection",
+  "data_schema": {
+    "fields": {
+      "new-response": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "lang_detection": {
+                  "dtype": "string",
+                  "signal": {
+                    "split_by_paragraph": false,
+                    "signal_name": "lang_detection"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "new-response",
+    "value",
+    "*"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-response/value/near_dup/signal_manifest.json

@@ -0,0 +1,41 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "new-response.value.near_dup",
+  "data_schema": {
+    "fields": {
+      "new-response": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "near_dup": {
+                  "fields": {
+                    "cluster_id": {
+                      "dtype": "uint32",
+                      "categorical": true
+                    }
+                  },
+                  "signal": {
+                    "threshold": 0.85,
+                    "signal_name": "near_dup"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "new-response",
+    "value",
+    "*"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-response/value/pii/signal_manifest.json

@@ -0,0 +1,50 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "new-response.value.pii",
+  "data_schema": {
+    "fields": {
+      "new-response": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "pii": {
+                  "fields": {
+                    "emails": {
+                      "repeated_field": {
+                        "dtype": "string_span"
+                      }
+                    },
+                    "ip_addresses": {
+                      "repeated_field": {
+                        "dtype": "string_span"
+                      }
+                    },
+                    "secrets": {
+                      "repeated_field": {
+                        "dtype": "string_span"
+                      }
+                    }
+                  },
+                  "signal": {
+                    "signal_name": "pii"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "new-response",
+    "value",
+    "*"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/new-response/value/text_statistics/signal_manifest.json

@@ -0,0 +1,64 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "new-response.value.text_statistics",
+  "data_schema": {
+    "fields": {
+      "new-response": {
+        "fields": {
+          "value": {
+            "repeated_field": {
+              "fields": {
+                "text_statistics": {
+                  "fields": {
+                    "num_characters": {
+                      "dtype": "int32"
+                    },
+                    "readability": {
+                      "dtype": "float32"
+                    },
+                    "log(type_token_ratio)": {
+                      "dtype": "float32"
+                    },
+                    "frac_non_ascii": {
+                      "dtype": "float32",
+                      "bins": [
+                        [
+                          "Low",
+                          null,
+                          0.15
+                        ],
+                        [
+                          "Medium",
+                          0.15,
+                          0.3
+                        ],
+                        [
+                          "High",
+                          0.3,
+                          null
+                        ]
+                      ]
+                    }
+                  },
+                  "signal": {
+                    "signal_name": "text_statistics"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "new-response",
+    "value",
+    "*"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/original-context/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee5c4ca43663633f531a587438913cc15fecad5baed5fdce2a1c7bc97a6e9260
+size 32775684

data/datasets/lilac/databricks-dolly-15k-curated-en/original-context/gte-small/signal_manifest.json

@@ -0,0 +1,32 @@
+{
+  "files": [],
+  "parquet_id": "original-context.gte-small",
+  "data_schema": {
+    "fields": {
+      "original-context": {
+        "fields": {
+          "gte-small": {
+            "repeated_field": {
+              "fields": {
+                "embedding": {
+                  "dtype": "embedding"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "signal_name": "gte-small"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "original-context"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/original-context/lang_detection/signal_manifest.json

@@ -0,0 +1,28 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "original-context.lang_detection",
+  "data_schema": {
+    "fields": {
+      "original-context": {
+        "fields": {
+          "lang_detection": {
+            "dtype": "string",
+            "signal": {
+              "split_by_paragraph": false,
+              "signal_name": "lang_detection"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "original-context"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/original-context/near_dup/signal_manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "original-context.near_dup",
+  "data_schema": {
+    "fields": {
+      "original-context": {
+        "fields": {
+          "near_dup": {
+            "fields": {
+              "cluster_id": {
+                "dtype": "uint32",
+                "categorical": true
+              }
+            },
+            "signal": {
+              "threshold": 0.85,
+              "signal_name": "near_dup"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "original-context"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/original-context/pii/signal_manifest.json

@@ -0,0 +1,42 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "original-context.pii",
+  "data_schema": {
+    "fields": {
+      "original-context": {
+        "fields": {
+          "pii": {
+            "fields": {
+              "emails": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "ip_addresses": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "secrets": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              }
+            },
+            "signal": {
+              "signal_name": "pii"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "original-context"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/original-context/text_statistics/signal_manifest.json

@@ -0,0 +1,56 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "original-context.text_statistics",
+  "data_schema": {
+    "fields": {
+      "original-context": {
+        "fields": {
+          "text_statistics": {
+            "fields": {
+              "num_characters": {
+                "dtype": "int32"
+              },
+              "readability": {
+                "dtype": "float32"
+              },
+              "log(type_token_ratio)": {
+                "dtype": "float32"
+              },
+              "frac_non_ascii": {
+                "dtype": "float32",
+                "bins": [
+                  [
+                    "Low",
+                    null,
+                    0.15
+                  ],
+                  [
+                    "Medium",
+                    0.15,
+                    0.3
+                  ],
+                  [
+                    "High",
+                    0.3,
+                    null
+                  ]
+                ]
+              }
+            },
+            "signal": {
+              "signal_name": "text_statistics"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "original-context"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/original-instruction/lang_detection/signal_manifest.json

@@ -0,0 +1,28 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "original-instruction.lang_detection",
+  "data_schema": {
+    "fields": {
+      "original-instruction": {
+        "fields": {
+          "lang_detection": {
+            "dtype": "string",
+            "signal": {
+              "split_by_paragraph": false,
+              "signal_name": "lang_detection"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "original-instruction"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/original-instruction/near_dup/signal_manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "original-instruction.near_dup",
+  "data_schema": {
+    "fields": {
+      "original-instruction": {
+        "fields": {
+          "near_dup": {
+            "fields": {
+              "cluster_id": {
+                "dtype": "uint32",
+                "categorical": true
+              }
+            },
+            "signal": {
+              "threshold": 0.85,
+              "signal_name": "near_dup"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "original-instruction"
+  ]
+}

data/datasets/lilac/databricks-dolly-15k-curated-en/original-instruction/pii/signal_manifest.json

@@ -0,0 +1,42 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "original-instruction.pii",
+  "data_schema": {
+    "fields": {
+      "original-instruction": {
+        "fields": {
+          "pii": {
+            "fields": {
+              "emails": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "ip_addresses": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "secrets": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              }
+            },
+            "signal": {
+              "signal_name": "pii"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "original-instruction"
+  ]
+}