auth

tools.py

@@ -41,13 +41,13 @@ memoires_ds = load_dataset("eliot-hub/memoires_vec_800", split="data", token=HF_
 batched_ds = memoires_ds.batch(batch_size=41000)
 client = chromadb.Client()
 collection = client.get_or_create_collection(name="embeddings_mxbai") 
-for batch in tqdm(batched_ds, desc="Processing dataset batches"): 
-    collection.add(
-        ids=batch["id"],
-        metadatas=batch["metadata"],
-        documents=batch["document"],
-        embeddings=batch["embedding"],      
-    )
+# for batch in tqdm(batched_ds, desc="Processing dataset batches"): 
+#     collection.add(
+#         ids=batch["id"],
+#         metadatas=batch["metadata"],
+#         documents=batch["document"],
+#         embeddings=batch["embedding"],      
+#     )
 print(f"Collection complete: {collection.count()}")
 del memoires_ds, batched_ds