hybrid embeddings

app.py

@@ -9,7 +9,10 @@ from qdrant_client import QdrantClient
 from langchain.retrievers import ContextualCompressionRetriever
 from langchain.retrievers.document_compressors import CrossEncoderReranker
 from langchain_community.cross_encoders import HuggingFaceCrossEncoder
+from langchain_qdrant import FastEmbedSparse, RetrievalMode
 
+
+# get the device to be used eithe gpu or cpu
 device = 'cuda' if cuda.is_available() else 'cpu'
 
 
@@ -20,70 +23,97 @@ var=st.text_input("enter keyword")
 
 def create_chunks(text):
     """TAKES A TEXT AND CERATES CREATES CHUNKS"""
+    # chunk size in terms of token
     text_splitter = TokenTextSplitter(chunk_size=500, chunk_overlap=0)
     texts = text_splitter.split_text(text)
     return texts 
 
 def get_chunks():
-    #orgas_df = pd.read_csv("iati_files/project_orgas.csv")
-    #region_df = pd.read_csv("iati_files/project_region.csv")
-    #sector_df = pd.read_csv("iati_files/project_sector.csv")
-    #status_df = pd.read_csv("iati_files/project_status.csv")
-    #texts_df = pd.read_csv("iati_files/project_texts.csv")
-
-    #projects_df = pd.merge(orgas_df, region_df, on='iati_id', how='inner')
-    #projects_df = pd.merge(projects_df, sector_df, on='iati_id', how='inner')
-    #projects_df = pd.merge(projects_df, status_df, on='iati_id', how='inner')
-    #projects_df = pd.merge(projects_df, texts_df, on='iati_id', how='inner')
-    #giz_df = projects_df[projects_df.client.str.contains('bmz')].reset_index(drop=True)
-
-    #giz_df.drop(columns= ['orga_abbreviation', 'client',
-    #    'orga_full_name', 'country', 
-    #    'country_flag', 'crs_5_code', 'crs_3_code',
-    #    'sgd_pred_code'], inplace=True)
-
-    giz_df = pd.read_json('iati_files/data_giz_website.json')
-    giz_df = giz_df.rename(columns={'content':'project_description'})
+    """
+    this will read the iati files and create the chunks
+    """
+    orgas_df = pd.read_csv("iati_files/project_orgas.csv")
+    region_df = pd.read_csv("iati_files/project_region.csv")
+    sector_df = pd.read_csv("iati_files/project_sector.csv")
+    status_df = pd.read_csv("iati_files/project_status.csv")
+    texts_df = pd.read_csv("iati_files/project_texts.csv")
+
+    projects_df = pd.merge(orgas_df, region_df, on='iati_id', how='inner')
+    projects_df = pd.merge(projects_df, sector_df, on='iati_id', how='inner')
+    projects_df = pd.merge(projects_df, status_df, on='iati_id', how='inner')
+    projects_df = pd.merge(projects_df, texts_df, on='iati_id', how='inner')
+    giz_df = projects_df[projects_df.client.str.contains('bmz')].reset_index(drop=True)
+
+    giz_df.drop(columns= ['orga_abbreviation', 'client',
+       'orga_full_name', 'country', 
+       'country_flag', 'crs_5_code', 'crs_3_code',
+       'sgd_pred_code'], inplace=True)
+
+    #### code for eading the giz_worldwide data
+    #giz_df = pd.read_json('iati_files/data_giz_website.json')
+    #giz_df = giz_df.rename(columns={'content':'project_description'})
     
 
-    giz_df['text_size'] = giz_df.apply(lambda x: len((x['project_name'] + x['project_description']).split()), axis=1)
-    giz_df['chunks'] = giz_df.apply(lambda x:create_chunks(x['project_name'] + x['project_description']),axis=1)
+    #giz_df['text_size'] = giz_df.apply(lambda x: len((x['project_name'] + x['project_description']).split()), axis=1)
+    #giz_df['chunks'] = giz_df.apply(lambda x:create_chunks(x['project_name'] + x['project_description']),axis=1)
+    #giz_df = giz_df.explode(column=['chunks'], ignore_index=True)
+
+
+    giz_df['text_size'] = giz_df.apply(lambda x: len((x['title_main'] + x['description_main']).split()), axis=1)
+    giz_df['chunks'] = giz_df.apply(lambda x:create_chunks(x['title_main'] + x['description_main']),axis=1)
     giz_df = giz_df.explode(column=['chunks'], ignore_index=True)
 
-        
     placeholder= []
     for i in range(len(giz_df)):
         placeholder.append(Document(page_content= giz_df.loc[i,'chunks'], 
-                                metadata={
-                                        "title_main":giz_df.loc[i,'title_main'],
-                                        "country_name":str(giz_df.loc[i,'countries']),
-                                        "client": giz_df_new.loc[i, 'client'],
-                                        "language":giz_df_new.loc[i, 'language'],
-                                        "political_sponsor":giz_df.loc[i, 'poli_trager'],
-                                        "url": giz_df.loc[i, 'url']
-                                        #"iati_id": giz_df.loc[i,'iati_id'],
-                                        #"iati_orga_id":giz_df.loc[i,'iati_orga_id'],
-                                        #"crs_5_name": giz_df.loc[i,'crs_5_name'],
-                                        #"crs_3_name": giz_df.loc[i,'crs_3_name'],
-                                        #"sgd_pred_str":giz_df.loc[i,'sgd_pred_str'],
-                                        #"status":giz_df.loc[i,'status'],
-                                        }))
+                                metadata={"iati_id": giz_df.loc[i,'iati_id'],
+                                        "iati_orga_id":giz_df.loc[i,'iati_orga_id'],
+                                        "country_name":str(giz_df.loc[i,'country_name']),
+                                        "crs_5_name": giz_df.loc[i,'crs_5_name'],
+                                        "crs_3_name": giz_df.loc[i,'crs_3_name'],
+                                        "sgd_pred_str":giz_df.loc[i,'sgd_pred_str'],
+                                        "status":giz_df.loc[i,'status'],
+                                        "title_main":giz_df.loc[i,'title_main'],}))
     return placeholder
 
+    # placeholder= []
+    # for i in range(len(giz_df)):
+    #     placeholder.append(Document(page_content= giz_df.loc[i,'chunks'], 
+    #                             metadata={
+    #                                     "title_main":giz_df.loc[i,'title_main'],
+    #                                     "country_name":str(giz_df.loc[i,'countries']),
+    #                                     "client": giz_df_new.loc[i, 'client'],
+    #                                     "language":giz_df_new.loc[i, 'language'],
+    #                                     "political_sponsor":giz_df.loc[i, 'poli_trager'],
+    #                                     "url": giz_df.loc[i, 'url']
+    #                                     #"iati_id": giz_df.loc[i,'iati_id'],
+    #                                     #"iati_orga_id":giz_df.loc[i,'iati_orga_id'],
+    #                                     #"crs_5_name": giz_df.loc[i,'crs_5_name'],
+    #                                     #"crs_3_name": giz_df.loc[i,'crs_3_name'],
+    #                                     #"sgd_pred_str":giz_df.loc[i,'sgd_pred_str'],
+    #                                     #"status":giz_df.loc[i,'status'],
+    #                                     }))
+    # return placeholder
+
 def embed_chunks(chunks):
+    """
+    takes the chunks and does the hybrid embedding for the list of chunks
+    """
     embeddings = HuggingFaceEmbeddings(
         model_kwargs = {'device': device},
         encode_kwargs = {'normalize_embeddings': True},
         model_name='BAAI/bge-m3'
     )
+    sparse_embeddings = FastEmbedSparse(model_name="Qdrant/bm25")
     # placeholder for collection
     print("starting embedding")
     qdrant_collections = {}
-    qdrant_collections['all'] = Qdrant.from_documents(
+    qdrant_collections['iati'] = Qdrant.from_documents(
                 chunks,
                 embeddings,
+                sparse_embeddings = sparse_embeddings,
                 path="/data/local_qdrant",
-                collection_name='all',
+                collection_name='iati',
             )
             
     print(qdrant_collections)
@@ -122,11 +152,19 @@ def get_context(vectorstore,query):
     print(f"retrieved paragraphs:{len(context_retrieved)}")
 
     return context_retrieved
-#chunks = get_chunks()
-vectorstores = get_local_qdrant() 
-vectorstore = vectorstores['all']
-button=st.button("search")
-results= get_context(vectorstore, f"find the relvant paragraphs for: {var}")
+
+# first we create the chunks for iati documents
+chunks = get_chunks()
+print("chunking done")
+
+# once the chunks are done, we perform hybrid emebddings
+qdrant_collections = embed_chunks(chunks)
+print(qdrant_collections.keys())
+
+# vectorstores = get_local_qdrant() 
+# vectorstore = vectorstores['all']
+# button=st.button("search")
+# results= get_context(vectorstore, f"find the relvant paragraphs for: {var}")
 if button:
     st.write(f"Found {len(results)} results for query:{var}")
 

iati_files/data_giz_website.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:be70c4b250aad01e53543bdd07c1d9f9fdd8a23be65e4a1d8c64f2272f2bbf03
-size 13980616

requirements.txt

@@ -7,4 +7,5 @@ langchain==0.1.20
 langsmith==0.1.99
 qdrant-client==1.10.1
 tiktoken
-torch==2.4.0
+torch==2.4.0
+fastembed