Upload folder using huggingface_hub

.gitattributes

@@ -102,3 +102,5 @@ data/datasets/lilac/wikitext-2-raw-v1/text/lang_detection/data-00000-of-00001.pa
 data/datasets/lilac/wikitext-2-raw-v1/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/wikitext-2-raw-v1/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/wikitext-2-raw-v1/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/opus100-en-us-validation/translation/en/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/opus100-en-us-validation/translation/es/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text

data/datasets/lilac/opus100-en-us-validation/config.yml

@@ -0,0 +1,32 @@
+embeddings:
+- embedding: gte-small
+  path: [translation, en]
+- embedding: gte-small
+  path: [translation, es]
+name: opus100-en-us-validation
+namespace: lilac
+settings:
+  preferred_embedding: gte-small
+  ui:
+    media_paths:
+    - [translation, es]
+    - [translation, en]
+signals:
+- path: [translation, es]
+  signal: {signal_name: near_dup}
+- path: [translation, es]
+  signal: {signal_name: pii}
+- path: [translation, es]
+  signal: {signal_name: lang_detection}
+- path: [translation, es]
+  signal: {signal_name: text_statistics}
+- path: [translation, en]
+  signal: {signal_name: near_dup}
+- path: [translation, en]
+  signal: {signal_name: text_statistics}
+- path: [translation, en]
+  signal: {signal_name: pii}
+- path: [translation, en]
+  signal: {signal_name: lang_detection}
+source: {config_name: en-es, dataset_name: opus100, source_name: huggingface, split: validation}
+tags: [machine-learning]

data/datasets/lilac/opus100-en-us-validation/manifest.json

@@ -0,0 +1,22 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "en": {
+            "dtype": "string"
+          },
+          "es": {
+            "dtype": "string"
+          }
+        }
+      },
+      "__hfsplit__": {
+        "dtype": "string"
+      }
+    }
+  }
+}

data/datasets/lilac/opus100-en-us-validation/translation/en/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e9362c74b9a4c6d10bd1bc3243d268876a85d1b6f829ef01fc88e4a8f562cff
+size 3397972

data/datasets/lilac/opus100-en-us-validation/translation/en/gte-small/signal_manifest.json

@@ -0,0 +1,37 @@
+{
+  "files": [],
+  "parquet_id": "translation.en.gte-small",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "en": {
+            "fields": {
+              "gte-small": {
+                "repeated_field": {
+                  "fields": {
+                    "embedding": {
+                      "dtype": "embedding"
+                    }
+                  },
+                  "dtype": "string_span"
+                },
+                "signal": {
+                  "signal_name": "gte-small"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "translation",
+    "en"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/lilac/opus100-en-us-validation/translation/en/lang_detection/signal_manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.en.lang_detection",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "en": {
+            "fields": {
+              "lang_detection": {
+                "dtype": "string",
+                "signal": {
+                  "split_by_paragraph": false,
+                  "signal_name": "lang_detection"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "translation",
+    "en"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/en/near_dup/signal_manifest.json

@@ -0,0 +1,38 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.en.near_dup",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "en": {
+            "fields": {
+              "near_dup": {
+                "fields": {
+                  "cluster_id": {
+                    "dtype": "uint32",
+                    "categorical": true
+                  }
+                },
+                "signal": {
+                  "threshold": 0.85,
+                  "signal_name": "near_dup"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "translation",
+    "en"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/en/pii/signal_manifest.json

@@ -0,0 +1,47 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.en.pii",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "en": {
+            "fields": {
+              "pii": {
+                "fields": {
+                  "emails": {
+                    "repeated_field": {
+                      "dtype": "string_span"
+                    }
+                  },
+                  "ip_addresses": {
+                    "repeated_field": {
+                      "dtype": "string_span"
+                    }
+                  },
+                  "secrets": {
+                    "repeated_field": {
+                      "dtype": "string_span"
+                    }
+                  }
+                },
+                "signal": {
+                  "signal_name": "pii"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "translation",
+    "en"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/en/text_statistics/signal_manifest.json

@@ -0,0 +1,61 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.en.text_statistics",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "en": {
+            "fields": {
+              "text_statistics": {
+                "fields": {
+                  "num_characters": {
+                    "dtype": "int32"
+                  },
+                  "readability": {
+                    "dtype": "float32"
+                  },
+                  "log(type_token_ratio)": {
+                    "dtype": "float32"
+                  },
+                  "frac_non_ascii": {
+                    "dtype": "float32",
+                    "bins": [
+                      [
+                        "Low",
+                        null,
+                        0.15
+                      ],
+                      [
+                        "Medium",
+                        0.15,
+                        0.3
+                      ],
+                      [
+                        "High",
+                        0.3,
+                        null
+                      ]
+                    ]
+                  }
+                },
+                "signal": {
+                  "signal_name": "text_statistics"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "translation",
+    "en"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/es/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:916f708d77ad590a289226863b983dbeb247a9d4400b9f938bd84f3fbbc338eb
+size 3421560

data/datasets/lilac/opus100-en-us-validation/translation/es/gte-small/signal_manifest.json

@@ -0,0 +1,37 @@
+{
+  "files": [],
+  "parquet_id": "translation.es.gte-small",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "es": {
+            "fields": {
+              "gte-small": {
+                "repeated_field": {
+                  "fields": {
+                    "embedding": {
+                      "dtype": "embedding"
+                    }
+                  },
+                  "dtype": "string_span"
+                },
+                "signal": {
+                  "signal_name": "gte-small"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "translation",
+    "es"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/lilac/opus100-en-us-validation/translation/es/lang_detection/signal_manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.es.lang_detection",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "es": {
+            "fields": {
+              "lang_detection": {
+                "dtype": "string",
+                "signal": {
+                  "split_by_paragraph": false,
+                  "signal_name": "lang_detection"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "translation",
+    "es"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/es/near_dup/signal_manifest.json

@@ -0,0 +1,38 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.es.near_dup",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "es": {
+            "fields": {
+              "near_dup": {
+                "fields": {
+                  "cluster_id": {
+                    "dtype": "uint32",
+                    "categorical": true
+                  }
+                },
+                "signal": {
+                  "threshold": 0.85,
+                  "signal_name": "near_dup"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "translation",
+    "es"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/es/pii/signal_manifest.json

@@ -0,0 +1,47 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.es.pii",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "es": {
+            "fields": {
+              "pii": {
+                "fields": {
+                  "emails": {
+                    "repeated_field": {
+                      "dtype": "string_span"
+                    }
+                  },
+                  "ip_addresses": {
+                    "repeated_field": {
+                      "dtype": "string_span"
+                    }
+                  },
+                  "secrets": {
+                    "repeated_field": {
+                      "dtype": "string_span"
+                    }
+                  }
+                },
+                "signal": {
+                  "signal_name": "pii"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "translation",
+    "es"
+  ]
+}

data/datasets/lilac/opus100-en-us-validation/translation/es/text_statistics/signal_manifest.json

@@ -0,0 +1,61 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "translation.es.text_statistics",
+  "data_schema": {
+    "fields": {
+      "translation": {
+        "fields": {
+          "es": {
+            "fields": {
+              "text_statistics": {
+                "fields": {
+                  "num_characters": {
+                    "dtype": "int32"
+                  },
+                  "readability": {
+                    "dtype": "float32"
+                  },
+                  "log(type_token_ratio)": {
+                    "dtype": "float32"
+                  },
+                  "frac_non_ascii": {
+                    "dtype": "float32",
+                    "bins": [
+                      [
+                        "Low",
+                        null,
+                        0.15
+                      ],
+                      [
+                        "Medium",
+                        0.15,
+                        0.3
+                      ],
+                      [
+                        "High",
+                        0.3,
+                        null
+                      ]
+                    ]
+                  }
+                },
+                "signal": {
+                  "signal_name": "text_statistics"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "translation",
+    "es"
+  ]
+}