Update auditqa/process_chunks.py

auditqa/process_chunks.py

@@ -67,13 +67,14 @@ def load_chunks():
     # placeholder for collection
     qdrant_collections = {}
     print("embeddings started")
-    batch_size = 10000  # Adjust this value based on your system's memory capacity
+    batch_size = 1000  # Adjust this value based on your system's memory capacity
     for i in range(0, len(chunks_list), batch_size):
-        print("embedding",(i+batch_size)/10000)
+        print("embedding",(i+batch_size)/1000)
         batch_docs = chunks_list[i:i+batch_size]
         qdrant = Qdrant.from_documents(
             batch_docs, embeddings,
             path="/data/local_qdrant",
+            recreate_collection=False,
             collection_name='reportsFeb2025',
         )