Update app.py

app.py

@@ -6,6 +6,7 @@ import requests
 import yaml
 import os
 from typing import Dict, List, Optional
+import re  
 
 @tool
 def fetch_news(topic: str, num_results: int = 5) -> List[Dict]:
@@ -70,10 +71,19 @@ def scrape_articles(articles: List[Dict]) -> List[Dict]:
     
     for article in articles:
         try:
+            # Skip known paywalled sites
+            domain = article['link'].lower()
+            if any(site in domain for site in ['nytimes.com', 'wsj.com', 'ft.com']):
+                article['full_content'] = f"Content not accessible - {article['source']} article requires subscription"
+                continue
+                
             full_content = webpage_tool.forward(article['link'])
-            article['full_content'] = full_content
+            if full_content and len(full_content.strip()) > 0:
+                article['full_content'] = full_content
+            else:
+                article['full_content'] = article['snippet']
         except Exception as e:
-            article['full_content'] = f"Failed to scrape content: {str(e)}"
+            article['full_content'] = article['snippet']
     
     return articles