Update app.py

app.py

@@ -1,5 +1,7 @@
 import streamlit as st
-import requests
+import pandas as pd
+from langchain_text_splitters import TokenTextSplitter
+from langchain.docstore.document import Document
 
 
 st.set_page_config(page_title="SEARCH IATI",layout='wide')
@@ -7,8 +9,52 @@ st.title("SEARCH IATI Database")
 var=st.text_input("write keyword")
 title = var.replace(' ','+')
 
+def create_chunks(text):
+    text_splitter = TokenTextSplitter(chunk_size=500, chunk_overlap=0)
+    texts = text_splitter.split_text(text)
+    return texts 
+
+def get_chunks():
+    orgas_df = pd.read_csv("iati_files/project_orgas.csv")
+    region_df = pd.read_csv("iati_files/project_region.csv")
+    sector_df = pd.read_csv("iati_files/project_sector.csv")
+    status_df = pd.read_csv("iati_files/project_status.csv")
+    texts_df = pd.read_csv("iati_files/project_texts.csv")
+
+    projects_df = pd.merge(orgas_df, region_df, on='iati_id', how='inner')
+    projects_df = pd.merge(projects_df, sector_df, on='iati_id', how='inner')
+    projects_df = pd.merge(projects_df, status_df, on='iati_id', how='inner')
+    projects_df = pd.merge(projects_df, texts_df, on='iati_id', how='inner')
+    giz_df = projects_df[projects_df.client.str.contains('bmz')].reset_index(drop=True)
+
+    giz_df.drop(columns= ['orga_abbreviation', 'client',
+        'orga_full_name', 'country', 
+        'country_flag', 'crs_5_code', 'crs_3_code',
+        'sgd_pred_code'], inplace=True)
+
+    giz_df['text_size'] = giz_df.apply(lambda x: len((x['title_main'] + x['description_main']).split()), axis=1)
+    giz_df['chunks'] = giz_df.apply(lambda x:create_chunks(x['title_main'] + x['description_main']),axis=1)
+    giz_df = giz_df.explode(column=['chunks'], ignore_index=True)
+
+        
+    placeholder= []
+    for i in range(len(giz_df)):
+        placeholder.append(Document(page_content= giz_df.loc[i,'chunks'], 
+                                metadata={"iati_id": giz_df.loc[i,'iati_id'],
+                                        "iati_orga_id":giz_df.loc[i,'iati_orga_id'],
+                                        "country_name":str(giz_df.loc[i,'country_name']),
+                                        "crs_5_name": giz_df.loc[i,'crs_5_name'],
+                                        "crs_3_name": giz_df.loc[i,'crs_3_name'],
+                                        "sgd_pred_str":giz_df.loc[i,'sgd_pred_str'],
+                                        "status":giz_df.loc[i,'status'],
+                                        "title_main":giz_df.loc[i,'title_main'],}))
+    return placeholder
+
+   
 
 button=st.button("search")
 
+chunks = get_chunks()
+
 if button :
-    st.write("Working on it")
+    st.write(placeholder[0])