Update auditqa/process_chunks.py

auditqa/process_chunks.py

@@ -69,23 +69,23 @@ def load_chunks():
     # placeholder for collection
     qdrant_collections = {}
     print("embeddings started")
-    batch_size = 1000  # Adjust this value based on your system's memory capacity
-    for i in range(0, len(chunks_list), batch_size):
-        print("embedding",(i+batch_size)/1000)
-        batch_docs = chunks_list[i:i+batch_size]
-        qdrant = Qdrant.from_documents(
-            batch_docs, embeddings,
-            path="/data/local_qdrant",
-            recreate_collection=False,
-            collection_name='reportsFeb2025',
-        )
+    #batch_size = 1000  # Adjust this value based on your system's memory capacity
+    #for i in range(0, len(chunks_list), batch_size):
+    #    print("embedding",(i+batch_size)/1000)
+    #    batch_docs = chunks_list[i:i+batch_size]
+    #    qdrant = Qdrant.from_documents(
+    #        batch_docs, embeddings,
+    #        path="/data/local_qdrant",
+    #       recreate_collection=False,
+    #        collection_name='reportsFeb2025',
+    #   )
         
-    #qdrant_collections['reportsFeb2025'] = Qdrant.from_documents(
-    #            chunks_list,
-    #            embeddings,
-    #            path="/data/local_qdrant",
-    #            collection_name='reportsFeb2025',
-    #        )
+    qdrant_collections['reportsFeb2025'] = Qdrant.from_documents(
+                chunks_list,
+                embeddings,
+                path="/data/local_qdrant",
+                collection_name='reportsFeb2025',
+            )
     print(qdrant_collections)
     print("vector embeddings done")
     return qdrant_collections