Update app.py

app.py

@@ -224,19 +224,31 @@ def google_news_search(term, num_results=5, lang="en", timeout=5, safe="active",
     return all_results
 
 def summarize_webpage(url, content, query, instructions, max_chars=1000):
-    # Preprocess the content
-    preprocessed_text = preprocess_web_content(content, query.split())
-    
+    if content is None:
+        return f"Unable to fetch or process content from {url}"
+
+    # Extract keywords from the query
+    keywords = query.split()
+
+    # Apply full preprocessing pipeline
+    preprocessed_text = preprocess_text(content)
+    preprocessed_text = remove_boilerplate(preprocessed_text)
+    filtered_text = keyword_filter(preprocessed_text, keywords)
+    summarized_text = summarize_text(filtered_text, num_sentences=5)  # Adjust num_sentences as needed
+
+    if not summarized_text:
+        return f"No relevant content found for the query in {url}"
+
     # Format a prompt for this specific webpage
     webpage_prompt = f"""
     Instructions: {instructions}
     Query: {query}
     URL: {url}
 
-    Webpage content:
-    {preprocessed_text}
+    Filtered and summarized webpage content:
+    {summarized_text}
 
-    Summarize the above content in relation to the query. Focus on relevant information and include any specific data or facts mentioned. Keep the summary concise, ideally under 200 words.
+    Based on the above filtered and summarized content, provide a concise summary that's directly relevant to the query. Focus on specific data, facts, or insights mentioned. Keep the summary under 200 words.
 
     Summary:
     """
@@ -248,11 +260,14 @@ def summarize_webpage(url, content, query, instructions, max_chars=1000):
     if summary and len(summary) > max_chars:
         summary = summary[:max_chars] + "..."
     
-    return summary
+    return summary if summary else f"Unable to generate summary for {url}"
 
 def preprocess_text(text):
+    if text is None:
+        return ""  # Return an empty string if input is None
+    
     # Remove HTML tags
-    text = BeautifulSoup(text, "html.parser").get_text()
+    text = BeautifulSoup(str(text), "html.parser").get_text()
     
     # Remove URLs
     text = re.sub(r'http\S+|www.\S+', '', text)
@@ -489,8 +504,12 @@ def scrape_and_display(query, num_results, instructions, web_search=True, use_ne
         # Summarize each result
         summarized_results = []
         for result in search_results:
-            summary = summarize_webpage(result['link'], result['text'], query, instructions)
-            summarized_results.append({"link": result['link'], "text": summary})
+            try:
+                summary = summarize_webpage(result['link'], result.get('text'), query, instructions)
+                summarized_results.append({"link": result['link'], "text": summary})
+            except Exception as e:
+                print(f"Error summarizing {result['link']}: {e}")
+                summarized_results.append({"link": result['link'], "text": f"Error summarizing content: {str(e)}"})
         
         formatted_prompt = format_prompt(query, summarized_results, instructions)
         generated_summary = generate_text(formatted_prompt, temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)