Update src/utils.py

src/utils.py

@@ -34,14 +34,13 @@ def has_meaningful_content(text):
 
 def remove_reasoning_and_sources(text):
     """
-    Remove reasoning and sources sections from the main response text.
-    Follow-up questions are no longer removed as they're now displayed separately.
+    Remove reasoning, follow-up questions, and sources sections from the main response text.
     
     Args:
         text (str): The text to clean
         
     Returns:
-        str: Text without reasoning and sources sections
+        str: Text without reasoning, follow-up questions, and sources sections
     """
     if not text:
         return text
@@ -50,8 +49,9 @@ def remove_reasoning_and_sources(text):
     pattern_reasoning = r'(?i)(\n+\s*reasoning:|\n+\s*\*{0,2}reasoning\*{0,2}:?|\n+\s*#{1,3}\s*reasoning).*?(?=\n+\s*(?:#{1,3}|follow[ -]?up questions:|sources:|references:|\Z))'
     cleaned_text = re.sub(pattern_reasoning, '', text, flags=re.DOTALL)
     
-    # Important: We'll keep follow-up questions in the main response to display inline
-    # We'll only extract them separately for the expandable section
+    # Remove follow-up questions sections
+    pattern_followup = r'(?i)(\n+\s*follow[ -]?up questions:|\n+\s*additional questions:|\n+\s*\*{0,2}follow[ -]?up questions\*{0,2}:?|\n+\s*#{1,3}\s*follow[ -]?up questions).*?(?=\n+\s*(?:#{1,3}|reasoning:|sources:|references:|\Z))'
+    cleaned_text = re.sub(pattern_followup, '', cleaned_text, flags=re.DOTALL)
     
     # Then, remove any sources/references sections
     pattern_sources = r'(?i)(\n+\s*sources:|\n+\s*references:|\n+\s*\*{0,2}sources\*{0,2}:?|\n+\s*\*{0,2}references\*{0,2}:?|\n+\s*#{1,3}\s*sources|\n+\s*#{1,3}\s*references).*?(?=\n+\s*(?:#{1,3}|\Z))'
@@ -66,8 +66,8 @@ def remove_reasoning_and_sources(text):
     skip_section = False
     
     for line in lines:
-        # Check if we should skip this line (part of reasoning or sources sections)
-        if re.search(r'(?i)^(\s*reasoning:|\s*sources:|\s*references:|\s*\*{0,2}reasoning\*{0,2}:?|\s*\*{0,2}sources\*{0,2}:?|\s*\*{0,2}references\*{0,2}:?|\s*#{1,3}\s*reasoning|\s*#{1,3}\s*sources|\s*#{1,3}\s*references)', line):
+        # Check if we should skip this line (part of reasoning, follow-up questions, or sources section)
+        if re.search(r'(?i)^(\s*reasoning:|\s*follow[ -]?up questions:|\s*additional questions:|\s*sources:|\s*references:|\s*\*{0,2}reasoning\*{0,2}:?|\s*\*{0,2}follow[ -]?up questions\*{0,2}:?|\s*\*{0,2}sources\*{0,2}:?|\s*\*{0,2}references\*{0,2}:?|\s*#{1,3}\s*reasoning|\s*#{1,3}\s*follow[ -]?up questions|\s*#{1,3}\s*sources|\s*#{1,3}\s*references)', line):
             skip_section = True
             continue
         # Check if we're entering a new section