Update rss_processor.py

rss_processor.py

@@ -8,8 +8,9 @@ from huggingface_hub import HfApi, login
 import shutil
 import rss_feeds
 from datetime import datetime
-import dateutil.parser  # For flexible date parsing
-import hashlib  # For generating unique hashes
+import dateutil.parser
+import hashlib
+import re  # For cleaning HTML and whitespace
 
 # Setup logging
 logging.basicConfig(level=logging.INFO)
@@ -38,6 +39,16 @@ vector_db = Chroma(
     collection_name=COLLECTION_NAME
 )
 
+def clean_text(text):
+    """Clean text by removing HTML tags and extra whitespace."""
+    if not text or not isinstance(text, str):
+        return ""
+    # Remove HTML tags
+    text = re.sub(r'<.*?>', '', text)
+    # Normalize whitespace (remove extra spaces, newlines, tabs)
+    text = ' '.join(text.split())
+    return text.strip().lower()
+
 def fetch_rss_feeds():
     articles = []
     seen_keys = set()
@@ -52,13 +63,18 @@ def fetch_rss_feeds():
             for entry in feed.entries:
                 if article_count >= MAX_ARTICLES_PER_FEED:
                     break
-                title = entry.get("title", "No Title").strip().lower()
-                link = entry.get("link", "").strip().lower()
-                description = entry.get("summary", entry.get("description", "No Description")).strip()
+                title = entry.get("title", "No Title")
+                link = entry.get("link", "")
+                description = entry.get("summary", entry.get("description", ""))
+                
+                # Clean and normalize all text fields
+                title = clean_text(title)
+                link = clean_text(link)
+                description = clean_text(description)
 
                 # Try multiple date fields and parse flexibly
                 published = "Unknown Date"
-                for date_field in ["published", "updated", "created"]:
+                for date_field in ["published", "updated", "created", "pubDate"]:  # Added "pubDate" for broader compatibility
                     if date_field in entry:
                         try:
                             parsed_date = dateutil.parser.parse(entry[date_field])
@@ -68,23 +84,23 @@ def fetch_rss_feeds():
                             logger.debug(f"Failed to parse {date_field} '{entry[date_field]}': {e}")
                             continue
 
-                # Use a robust key for deduplication, including a hash of the description
-                description_hash = hashlib.md5(description.encode('utf-8')).hexdigest()
+                # Use a robust key for deduplication, including cleaned fields
+                description_hash = hashlib.sha256(description.encode('utf-8')).hexdigest()  # Switched to SHA256 for better uniqueness
                 key = f"{title}|{link}|{published}|{description_hash}"
                 if key not in seen_keys:
                     seen_keys.add(key)
                     # Try multiple image sources
                     image = "svg"  # Default fallback
                     for img_source in [
-                        lambda e: e.get("media_content", [{}])[0].get("url"),
-                        lambda e: e.get("media_thumbnail", [{}])[0].get("url"),
-                        lambda e: e.get("enclosure", {}).get("url"),
-                        lambda e: next((lnk.get("href") for lnk in e.get("links", []) if lnk.get("type", "").startswith("image")), None),
+                        lambda e: clean_text(e.get("media_content", [{}])[0].get("url")) if e.get("media_content") else "",
+                        lambda e: clean_text(e.get("media_thumbnail", [{}])[0].get("url")) if e.get("media_thumbnail") else "",
+                        lambda e: clean_text(e.get("enclosure", {}).get("url")) if e.get("enclosure") else "",
+                        lambda e: clean_text(next((lnk.get("href") for lnk in e.get("links", []) if lnk.get("type", "").startswith("image")), "")),
                     ]:
                         try:
                             img = img_source(entry)
-                            if img and isinstance(img, str) and img.strip():
-                                image = img.strip()
+                            if img and img.strip():
+                                image = img
                                 break
                         except (IndexError, AttributeError, TypeError):
                             continue
@@ -106,33 +122,33 @@ def fetch_rss_feeds():
     return articles
 
 def categorize_feed(url):
-    if "nature" in url or "science.org" in url or "arxiv.org" in url or "plos.org" in url or "annualreviews.org" in url or "journals.uchicago.edu" in url or "jneurosci.org" in url or "cell.com" in url or "nejm.org" in url or "lancet.com" in url:
+    if "nature" in url.lower() or "science.org" in url.lower() or "arxiv.org" in url.lower() or "plos.org" in url.lower() or "annualreviews.org" in url.lower() or "journals.uchicago.edu" in url.lower() or "jneurosci.org" in url.lower() or "cell.com" in url.lower() or "nejm.org" in url.lower() or "lancet.com" in url.lower():
         return "Academic Papers"
-    elif "reuters.com/business" in url or "bloomberg.com" in url or "ft.com" in url or "marketwatch.com" in url or "cnbc.com" in url or "foxbusiness.com" in url or "wsj.com" in url or "bworldonline.com" in url or "economist.com" in url or "forbes.com" in url:
+    elif "reuters.com/business" in url.lower() or "bloomberg.com" in url.lower() or "ft.com" in url.lower() or "marketwatch.com" in url.lower() or "cnbc.com" in url.lower() or "foxbusiness.com" in url.lower() or "wsj.com" in url.lower() or "bworldonline.com" in url.lower() or "economist.com" in url.lower() or "forbes.com" in url.lower():
         return "Business"
-    elif "investing.com" in url or "cnbc.com/market" in url or "marketwatch.com/market" in url or "fool.co.uk" in url or "zacks.com" in url or "seekingalpha.com" in url or "barrons.com" in url or "yahoofinance.com" in url:
+    elif "investing.com" in url.lower() or "cnbc.com/market" in url.lower() or "marketwatch.com/market" in url.lower() or "fool.co.uk" in url.lower() or "zacks.com" in url.lower() or "seekingalpha.com" in url.lower() or "barrons.com" in url.lower() or "yahoofinance.com" in url.lower():
         return "Stocks & Markets"
-    elif "whitehouse.gov" in url or "state.gov" in url or "commerce.gov" in url or "transportation.gov" in url or "ed.gov" in url or "dol.gov" in url or "justice.gov" in url or "federalreserve.gov" in url or "occ.gov" in url or "sec.gov" in url or "bls.gov" in url or "usda.gov" in url or "gao.gov" in url or "cbo.gov" in url or "fema.gov" in url or "defense.gov" in url or "hhs.gov" in url or "energy.gov" in url or "interior.gov" in url:
+    elif "whitehouse.gov" in url.lower() or "state.gov" in url.lower() or "commerce.gov" in url.lower() or "transportation.gov" in url.lower() or "ed.gov" in url.lower() or "dol.gov" in url.lower() or "justice.gov" in url.lower() or "federalreserve.gov" in url.lower() or "occ.gov" in url.lower() or "sec.gov" in url.lower() or "bls.gov" in url.lower() or "usda.gov" in url.lower() or "gao.gov" in url.lower() or "cbo.gov" in url.lower() or "fema.gov" in url.lower() or "defense.gov" in url.lower() or "hhs.gov" in url.lower() or "energy.gov" in url.lower() or "interior.gov" in url.lower():
         return "Federal Government"
-    elif "weather.gov" in url or "metoffice.gov.uk" in url or "accuweather.com" in url or "weatherunderground.com" in url or "noaa.gov" in url or "wunderground.com" in url or "climate.gov" in url or "ecmwf.int" in url or "bom.gov.au" in url:
+    elif "weather.gov" in url.lower() or "metoffice.gov.uk" in url.lower() or "accuweather.com" in url.lower() or "weatherunderground.com" in url.lower() or "noaa.gov" in url.lower() or "wunderground.com" in url.lower() or "climate.gov" in url.lower() or "ecmwf.int" in url.lower() or "bom.gov.au" in url.lower():
         return "Weather"
-    elif "data.worldbank.org" in url or "imf.org" in url or "un.org" in url or "oecd.org" in url or "statista.com" in url or "kff.org" in url or "who.int" in url or "cdc.gov" in url or "bea.gov" in url or "census.gov" in url or "fdic.gov" in url:
+    elif "data.worldbank.org" in url.lower() or "imf.org" in url.lower() or "un.org" in url.lower() or "oecd.org" in url.lower() or "statista.com" in url.lower() or "kff.org" in url.lower() or "who.int" in url.lower() or "cdc.gov" in url.lower() or "bea.gov" in url.lower() or "census.gov" in url.lower() or "fdic.gov" in url.lower():
         return "Data & Statistics"
-    elif "nasa" in url or "spaceweatherlive" in url or "space" in url or "universetoday" in url or "skyandtelescope" in url or "esa" in url:
+    elif "nasa" in url.lower() or "spaceweatherlive" in url.lower() or "space" in url.lower() or "universetoday" in url.lower() or "skyandtelescope" in url.lower() or "esa" in url.lower():
         return "Space"
-    elif "sciencedaily" in url or "quantamagazine" in url or "smithsonianmag" in url or "popsci" in url or "discovermagazine" in url or "scientificamerican" in url or "newscientist" in url or "livescience" in url or "atlasobscura" in url:
+    elif "sciencedaily" in url.lower() or "quantamagazine" in url.lower() or "smithsonianmag" in url.lower() or "popsci" in url.lower() or "discovermagazine" in url.lower() or "scientificamerican" in url.lower() or "newscientist" in url.lower() or "livescience" in url.lower() or "atlasobscura" in url.lower():
         return "Science"
-    elif "wired" in url or "techcrunch" in url or "arstechnica" in url or "gizmodo" in url or "theverge" in url:
+    elif "wired" in url.lower() or "techcrunch" in url.lower() or "arstechnica" in url.lower() or "gizmodo" in url.lower() or "theverge" in url.lower():
         return "Tech"
-    elif "horoscope" in url or "astrostyle" in url:
+    elif "horoscope" in url.lower() or "astrostyle" in url.lower():
         return "Astrology"
-    elif "cnn_allpolitics" in url or "bbci.co.uk/news/politics" in url or "reuters.com/arc/outboundfeeds/newsletter-politics" in url or "politico.com/rss/politics" in url or "thehill" in url:
+    elif "cnn_allpolitics" in url.lower() or "bbci.co.uk/news/politics" in url.lower() or "reuters.com/arc/outboundfeeds/newsletter-politics" in url.lower() or "politico.com/rss/politics" in url.lower() or "thehill" in url.lower():
         return "Politics"
-    elif "weather" in url or "swpc.noaa.gov" in url or "foxweather" in url:
+    elif "weather" in url.lower() or "swpc.noaa.gov" in url.lower() or "foxweather" in url.lower():
         return "Earth Weather"
-    elif "vogue" in url:
+    elif "vogue" in url.lower():
         return "Lifestyle"
-    elif "phys.org" in url or "aps.org" in url or "physicsworld" in url:
+    elif "phys.org" in url.lower() or "aps.org" in url.lower() or "physicsworld" in url.lower():
         return "Physics"
     return "Uncategorized"
 
@@ -141,12 +157,12 @@ def process_and_store_articles(articles):
     existing_ids = set(vector_db.get()["ids"])  # Get existing document IDs to avoid duplicates
     for article in articles:
         try:
-            # Create a unique ID based on normalized fields, including description hash
-            title = article["title"].lower().strip()
-            link = article["link"].lower().strip()
-            description = article["description"].strip()
+            # Clean and normalize all fields
+            title = clean_text(article["title"])
+            link = clean_text(article["link"])
+            description = clean_text(article["description"])
             published = article["published"]
-            description_hash = hashlib.md5(description.encode('utf-8')).hexdigest()
+            description_hash = hashlib.sha256(description.encode('utf-8')).hexdigest()
             doc_id = f"{title}|{link}|{published}|{description_hash}"
             if doc_id in existing_ids:
                 logger.debug(f"Skipping duplicate in DB: {doc_id}")