cleaned rag documents

conversation.py

@@ -136,8 +136,16 @@ import time
 from db_func import insert_one
 from langchain.agents import AgentExecutor
 import re
+import wordninja
 
+def clean_text(text):
 
+    text = text.strip().lower()
+    utput_paragraph = ' '.join(''.join(input_paragraph.split()).split(' '))
+    words = wordninja.split(output_paragraph)
+
+    return ' '.join(words)
+    
 def get_bert_embeddings(sentence):
     embeddings = []
     input_ids = tokenizer.encode(sentence, return_tensors="pt")
@@ -214,7 +222,7 @@ def run(input_):
         for i in range(len(sources)):
             temp = sources[i].replace('.pdf',  '').replace('.txt', '').replace("AAO", "").replace("2022-2023", "").replace("data/book", "").replace("text", "").replace("  ", " ")
             source_text += f"{i+1}. {temp}\n"
-            cleaned_text = re.sub(r'[^a-zA-Z0-9\s]', '', docs[i].strip().lower())
+            cleaned_text = re.sub(r'[^a-zA-Z0-9\s]', '', clean_text(docs[i]))
             doc_text += f"{i+1}. {cleaned_text}\n"
 
         # output_text =  f"{output_text} \n\nSources: \n{source_text}"