batch

app.py

@@ -57,7 +57,7 @@ with gr.Blocks(theme=theme, js=js_func, title="Dataltist", fill_height=True) as
     gr.Markdown("# Dataltist Chatbot 🚀")
     chatbot = gr.Chatbot(show_copy_button=True, show_share_button=False, type="messages", scale=1)
     msg = gr.Textbox(lines=1, show_label=False, placeholder="Posez vos questions sur l'assurance")  #  submit_btn=True
-    clear = gr.ClearButton([msg, chatbot], value="Effacer 🗑")
+    # clear = gr.ClearButton([msg, chatbot], value="Effacer 🗑")
     config = {"configurable": {"thread_id": "1"}} 
 
 

tools.py

@@ -18,7 +18,8 @@ import chromadb
 from typing import List
 from datasets import load_dataset
 from langchain_huggingface import HuggingFaceEmbeddings 
-
+from tqdm import tqdm
+import datetime
 
 load_dotenv()
 # Global params
@@ -40,8 +41,15 @@ memoires_ds = load_dataset("eliot-hub/memoires_vec_800", split="data", token=HF_
 batched_ds = memoires_ds.batch(batch_size=41000)
 client = chromadb.Client()
 collection = client.get_or_create_collection(name="embeddings_mxbai") 
-
-
+for batch in tqdm(batched_ds, desc=f"[{datetime.now().strftime('%H:%M:%S')}] Processing dataset batches"):
+    collection.add(
+        ids=batch["ids"],
+        # metadatas=batch["metadatas"],
+        documents=batch["documents"],
+        embeddings=batch["embeddings"],      
+    )
+print("DB done")
+del memoires_ds, batched_ds
 
 llm_4o = ChatOpenAI(model="gpt-4o-mini", api_key=OPENAI_API_KEY, temperature=0)