Update auditqa/process_chunks.py

auditqa/process_chunks.py

@@ -49,48 +49,15 @@ def load_chunks():
     # 'source'=='category', 'subtype', these are used in UI for document selection
     #  which will be used later for filtering database
     config = getconfig("./model_params.cfg")
-    all_documents = {}
-    categories = list(files.keys())
-    # iterate through 'source'
-    for category in categories:
-        print("documents splitting in source:",category)
-        all_documents[category] = []
-        subtypes = list(files[category].keys())
-        # iterate through 'subtype' within the source
-        # example source/category == 'District', has subtypes which is district names
-        for subtype in subtypes:
-            print("document splitting for subtype:",subtype)
-            for file in files[category][subtype]:
-
-                # load the chunks
-                try:
-                    doc_processed = open_file(path_to_data + file + "/"+ file+ ".chunks.json" )
-
-                
-                except Exception as e:
-                    print("Exception: ", e)
-                print("chunks in subtype:",subtype, "are:",len(doc_processed))
-
-                # add metadata information 
-                chunks_list = []
-                for doc in doc_processed:
-                    chunks_list.append(Document(page_content= doc['content'], 
-                             metadata={"source": category,
-                                      "subtype":subtype,
-                                      "year":file[-4:],
-                                      "filename":file,
-                                      "page":doc['metadata']['page'],
-                                      "headings":doc['metadata']['headings']}))
-
-                all_documents[category].append(chunks_list)
-    
-    # convert list of list to flat list
-    for key, docs_processed in all_documents.items():
-        docs_processed = [item for sublist in docs_processed for item in sublist]
-        print("length of chunks in source:",key, "are:",len(docs_processed))
-        all_documents[key] = docs_processed
-    all_documents['allreports'] = [sublist for key,sublist in all_documents.items()]
-    all_documents['allreports'] = [item for sublist in all_documents['allreports'] for item in sublist]
+  
+    doc_processed = open_file(path_to_data + "new_chunks.json" ) 
+    chunks_list = []
+
+    for doc in doc_processed:
+        chunks_list.append(Document(page_content= doc['content'], 
+                 metadata=doc['metadata']
+                                    ))
+
     # define embedding model
     embeddings = HuggingFaceEmbeddings(
         model_kwargs = {'device': device},
@@ -99,16 +66,12 @@ def load_chunks():
     )
     # placeholder for collection
     qdrant_collections = {}
-    
-    
-    for file,value in all_documents.items():
-        if file == "allreports":
-            print("emebddings for:",file)
-            qdrant_collections[file] = Qdrant.from_documents(
-                value,
+    print("embeddings started")
+    qdrant_collections['reportsFeb2025'] = Qdrant.from_documents(
+                chunks_list,
                 embeddings,
                 path="/data/local_qdrant",
-                collection_name=file,
+                collection_name='reportsFeb2025',
             )
     print(qdrant_collections)
     print("vector embeddings done")