Update app.py

app.py

@@ -439,20 +439,53 @@ def bot(history, choice, tts_choice, retrieval_mode, model_choice):
 
 import re
 
-def clean_response(response_text):
-    # Remove system and user tags
-    response_text = re.sub(r'<\|system\|>.*?<\|end\|>', '', response_text, flags=re.DOTALL)
-    response_text = re.sub(r'<\|user\|>.*?<\|end\|>', '', response_text, flags=re.DOTALL)
-    response_text = re.sub(r'<\|assistant\|>', '', response_text, flags=re.DOTALL)
+# def clean_response(response_text):
+#     # Remove system and user tags
+#     response_text = re.sub(r'<\|system\|>.*?<\|end\|>', '', response_text, flags=re.DOTALL)
+#     response_text = re.sub(r'<\|user\|>.*?<\|end\|>', '', response_text, flags=re.DOTALL)
+#     response_text = re.sub(r'<\|assistant\|>', '', response_text, flags=re.DOTALL)
+
+#     # Clean up the text by removing extra whitespace
+#     cleaned_response = response_text.strip()
+#     cleaned_response = re.sub(r'\s+', ' ', cleaned_response)
+
+#     # Ensure the response is conversational and organized
+#     cleaned_response = cleaned_response.replace('1.', '\n1.').replace('2.', '\n2.').replace('3.', '\n3.').replace('4.', '\n4.').replace('5.', '\n5.')
+
+#     return cleaned_response
+
+
+
+import re
 
-    # Clean up the text by removing extra whitespace
-    cleaned_response = response_text.strip()
-    cleaned_response = re.sub(r'\s+', ' ', cleaned_response)
+def clean_response(response_text):
+    """
+    This function removes metadata and unnecessary symbols from the document response
+    and formats the output in a readable way.
+    """
+    # Remove metadata section from the response
+    response_text = re.sub(r'Document\(metadata=.*?,page_content="', '', response_text)
+    
+    # Replace encoded characters
+    response_text = response_text.replace('\\u2019', "'")  # replace unicode apostrophe
+    response_text = response_text.replace('\\u00e8', 'è')  # replace accented characters
+    response_text = response_text.replace('\\u00e0', 'à')
+    response_text = response_text.replace('\\n', '\n')  # newline characters
+    response_text = response_text.replace('\\\\', '\\')  # backslashes
+    
+    # Remove any trailing document information
+    response_text = re.sub(r'\\.*$', '', response_text) 
+    
+    # Ensure proper spacing for better readability
+    response_text = re.sub(r'([a-z])([A-Z])', r'\1 \2', response_text)  # Add spaces between words joined together
+    
+    # Properly format new lines and spacing
+    response_text = response_text.strip()  # Remove leading/trailing whitespace
+    response_text = re.sub(r' +', ' ', response_text)  # Replace multiple spaces with a single space
+    response_text = re.sub(r'\n+', '\n', response_text)  # Replace multiple newlines with a single newline
 
-    # Ensure the response is conversational and organized
-    cleaned_response = cleaned_response.replace('1.', '\n1.').replace('2.', '\n2.').replace('3.', '\n3.').replace('4.', '\n4.').replace('5.', '\n5.')
+    return response_text
 
-    return cleaned_response
 
 # Define a new template specifically for GPT-4o-mini in VDB Details mode
 gpt4o_mini_template_details = f"""