batch

app.py

@@ -65,6 +65,7 @@ collection = import_into_chroma(
     dataset=Memoires_DS,
     embedding_function=huggingface_ef #Memoires_DS.embedding_function
     )
+del Memoires_DS
 db = Chroma(
     client=client,
     collection_name=f"embeddings_mxbai",

hf_to_chroma_ds.py

@@ -69,7 +69,7 @@ class Memoires_DS(Dataset):
 
 
 
-def import_into_chroma(chroma_client, dataset, collection_name=None, embedding_function=None, batch_size=5000):
+def import_into_chroma(chroma_client, dataset, collection_name=None, embedding_function=None, batch_size=10000):
     """
     Imports a dataset into Chroma in batches.