Spaces:
Sleeping
Sleeping
batch_ids
Browse files
tools.py
CHANGED
@@ -41,14 +41,14 @@ memoires_ds = load_dataset("eliot-hub/memoires_vec_800", split="data", token=HF_
|
|
41 |
batched_ds = memoires_ds.batch(batch_size=41000)
|
42 |
client = chromadb.Client()
|
43 |
collection = client.get_or_create_collection(name="embeddings_mxbai")
|
44 |
-
for batch in tqdm(batched_ds, desc=
|
45 |
collection.add(
|
46 |
-
ids=batch["
|
47 |
-
|
48 |
-
documents=batch["
|
49 |
-
embeddings=batch["
|
50 |
)
|
51 |
-
print("
|
52 |
del memoires_ds, batched_ds
|
53 |
|
54 |
llm_4o = ChatOpenAI(model="gpt-4o-mini", api_key=OPENAI_API_KEY, temperature=0)
|
|
|
41 |
batched_ds = memoires_ds.batch(batch_size=41000)
|
42 |
client = chromadb.Client()
|
43 |
collection = client.get_or_create_collection(name="embeddings_mxbai")
|
44 |
+
for batch in tqdm(batched_ds, desc="Processing dataset batches"):
|
45 |
collection.add(
|
46 |
+
ids=batch["id"],
|
47 |
+
metadatas=batch["metadata"],
|
48 |
+
documents=batch["document"],
|
49 |
+
embeddings=batch["embedding"],
|
50 |
)
|
51 |
+
print(f"Collection complete: {collection.count()}")
|
52 |
del memoires_ds, batched_ds
|
53 |
|
54 |
llm_4o = ChatOpenAI(model="gpt-4o-mini", api_key=OPENAI_API_KEY, temperature=0)
|