added rag documents

conversation.py

@@ -135,6 +135,7 @@ import gradio as gr
 import time
 from db_func import insert_one
 from langchain.agents import AgentExecutor
+import re
 
 
 def get_bert_embeddings(sentence):
@@ -213,9 +214,11 @@ def run(input_):
         for i in range(len(sources)):
             temp = sources[i].replace('.pdf',  '').replace('.txt', '').replace("AAO", "").replace("2022-2023", "").replace("data/book", "").replace("text", "").replace("  ", " ")
             source_text += f"{i+1}. {temp}\n"
-            doc_text += f"{i+1}. {docs[i]}\n"
+            cleaned_text = re.sub(r'[^a-zA-Z0-9\s]', '', docs[i].strip().lower())
+            doc_text += f"{i+1}. {cleaned_text}\n"
 
-        output_text =  f"{output_text} \n\nSources: \n{source_text}"
+        # output_text =  f"{output_text} \n\nSources: \n{source_text}"
+        output_text =  f"{output_text} \n\nSources: \n{source_text}\n\nDocuments: \n{doc_text}"
         # output_text =  f"{output_text}"
     
     doc_to_insert = {