Adding new widget to display data sources from where indices have been created

app.py

@@ -1,6 +1,7 @@
 import gradio as gr
 import os
 import datetime
+import json
 
 import src.constants as constants_utils
 import src.kkms_kssw as kkms_kssw
@@ -32,6 +33,7 @@ class DomState:
         self.weather_forecast = ''
         self.weather_forecast_summary = ''
         self.indic_translation = ''
+        self.kb_sources = {}
         
         # Initialize index (vector store) - This will create a new index from scratch if load_from_existing_index_file == False
         self.kkms_kssw_obj = kkms_kssw.KKMS_KSSW()
@@ -160,6 +162,30 @@ class DomState:
         return self.kkms_kssw_obj.weather_utils_obj.get_weather_forecast(state, district)
 
 
+    def click_handler_for_get_kb_sources(
+        self
+    ):
+        def _serialize_sets(obj):
+            if isinstance(obj, set):
+                return list(obj)
+            return obj
+        self.kb_sources = self.kkms_kssw_obj.langchain_utils_obj.get_index_category_wise_data_sources()
+        # return json.dumps(self.kb_sources, default=_serialize_sets)
+        kb_sources = ''
+        for index_category, doc_type in self.kb_sources.items():
+            kb_sources = '\n' + f'='*100 + '\n'
+            kb_sources += f'Question Category: {index_category}'
+            for dt, source in doc_type.items():
+                kb_sources += '\n' + f'='*50 + '\n'
+                if not source:
+                    continue
+                kb_sources += f'Document type: {dt}'
+                for doc in source:
+                    kb_sources += f'\t\t\t\n{doc}'
+        self.kb_sources = kb_sources
+        return self.kb_sources
+
+
     def _upload_file(self, files):
         file_paths = [file.name for file in files]
         return file_paths
@@ -176,6 +202,7 @@ class DomState:
                 gr.update(visible=False),
                 gr.update(visible=False),
                 gr.update(visible=False),
+                gr.update(visible=False),
             ]
 
         elif choice == "General (AgGPT)":
@@ -185,6 +212,7 @@ class DomState:
                 gr.update(visible=False),
                 gr.update(visible=False),
                 gr.update(visible=False),
+                gr.update(visible=False),
             ]
         
         elif choice == "Mandi Price":
@@ -194,6 +222,7 @@ class DomState:
                 gr.update(visible=True),
                 gr.update(visible=False),
                 gr.update(visible=False),
+                gr.update(visible=False),
             ]
         
         elif choice == "Weather":
@@ -203,10 +232,22 @@ class DomState:
                 gr.update(visible=False),
                 gr.update(visible=True),
                 gr.update(visible=False),
+                gr.update(visible=False),
             ]
 
         elif choice == "Load Custom Data":
             return [
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=True),
+                gr.update(visible=False),
+            ]
+
+        elif choice == "Display Data Sources":
+            return [
+                gr.update(visible=False),
                 gr.update(visible=False),
                 gr.update(visible=False),
                 gr.update(visible=False),
@@ -276,7 +317,8 @@ with gr.Blocks(title='KKMS-Smart-Search-Demo') as demo:
             "General (AgGPT)",
             "Mandi Price",
             "Weather",
-            "Load Custom Data"
+            "Load Custom Data",
+            "Display Data Sources",
         ],
         label="Query related to",
         value="Custom Query"
@@ -519,6 +561,19 @@ with gr.Blocks(title='KKMS-Smart-Search-Demo') as demo:
                         rowUploadUrls,
                     ],
                 )
+
+
+    #############################################################################
+    # Widget to display what all PDFs/Text files, URLs are ingested and indexed for querying in the KB (Knowledge Base)
+    with gr.Row(visible=False) as rowDisplayDataSources:
+        with gr.Column(scale=1, min_width=600):
+            with gr.Tab(label='Following PDFs, Text files, and URLs have been ingested and indexed in the Knowledge Base and are available for querying.'):
+                kb_sources = gr.Textbox(label=f"Data loaded from:", value=dom.kb_sources, interactive=False)
+                b_kb_sources = gr.Button("Display Data Sources").style(size='sm')
+                b_kb_sources.click(
+                    fn=dom.click_handler_for_get_kb_sources,
+                    outputs=kb_sources
+                )
     
     
     widgets.change(
@@ -530,6 +585,7 @@ with gr.Blocks(title='KKMS-Smart-Search-Demo') as demo:
             rowMandiPrice,
             rowWeather,
             rowLoadCustomData,
+            rowDisplayDataSources,
         ],
     )
 

src/langchain_utils.py

@@ -75,6 +75,9 @@ class LANGCHAIN_UTILS:
             }
         ]
 
+        # Index category - doc_type wise data sources to display in widget
+        self.index_category_doc_type_wise_data_sources = {}
+
 
     def generate_prompt_template(
         self,
@@ -889,3 +892,22 @@ class LANGCHAIN_UTILS:
 
         # Or just nuke the persist directory
         # !rm -rf self.index_filepath
+
+
+    def get_index_category_wise_data_sources(
+        self
+    ):
+        # self.index_category_doc_type_wise_data_sources
+        for index_category, doc_type in self.index_category_doc_type_wise_index.items():
+            self.index_category_doc_type_wise_data_sources.setdefault(index_category, {})
+            for dt in doc_type.keys():
+                if dt == 'master':
+                    continue
+                self.index_category_doc_type_wise_data_sources[index_category].setdefault(dt, set())
+                if doc_type[dt]:
+                    docs = doc_type[dt].docstore._dict
+                    for doc, val in docs.items():
+                        if 'source' in val.metadata and val.metadata['source']:
+                            self.index_category_doc_type_wise_data_sources[index_category][dt].add(val.metadata['source'])
+        
+        return self.index_category_doc_type_wise_data_sources