Update rss_processor.py

rss_processor.py

@@ -9,7 +9,7 @@ import shutil
 import rss_feeds
 from datetime import datetime
 import dateutil.parser  # For flexible date parsing
-import hashlib
+import hashlib  # For generating unique hashes
 
 # Setup logging
 logging.basicConfig(level=logging.INFO)
@@ -52,7 +52,7 @@ def fetch_rss_feeds():
             for entry in feed.entries:
                 if article_count >= MAX_ARTICLES_PER_FEED:
                     break
-                title = entry.get("title", "No Title").strip().lower()  # Normalize case and whitespace
+                title = entry.get("title", "No Title").strip().lower()
                 link = entry.get("link", "").strip().lower()
                 description = entry.get("summary", entry.get("description", "No Description")).strip()
 
@@ -68,10 +68,9 @@ def fetch_rss_feeds():
                             logger.debug(f"Failed to parse {date_field} '{entry[date_field]}': {e}")
                             continue
 
-                # Use a robust key for deduplication
-                #key = f"{title}|{link}|{published}"
-                key = f"{title.lower()}|{link.lower()}|{published}|{hash(description.encode('utf-8'))}"
-
+                # Use a robust key for deduplication, including a hash of the description
+                description_hash = hashlib.md5(description.encode('utf-8')).hexdigest()
+                key = f"{title}|{link}|{published}|{description_hash}"
                 if key not in seen_keys:
                     seen_keys.add(key)
                     # Try multiple image sources
@@ -107,7 +106,6 @@ def fetch_rss_feeds():
     return articles
 
 def categorize_feed(url):
-    # (Unchanged, keeping your existing categorization logic)
     if "nature" in url or "science.org" in url or "arxiv.org" in url or "plos.org" in url or "annualreviews.org" in url or "journals.uchicago.edu" in url or "jneurosci.org" in url or "cell.com" in url or "nejm.org" in url or "lancet.com" in url:
         return "Academic Papers"
     elif "reuters.com/business" in url or "bloomberg.com" in url or "ft.com" in url or "marketwatch.com" in url or "cnbc.com" in url or "foxbusiness.com" in url or "wsj.com" in url or "bworldonline.com" in url or "economist.com" in url or "forbes.com" in url:
@@ -143,8 +141,13 @@ def process_and_store_articles(articles):
     existing_ids = set(vector_db.get()["ids"])  # Get existing document IDs to avoid duplicates
     for article in articles:
         try:
-            # Create a unique ID based on normalized fields
-            doc_id = f"{article['title'].lower()}|{article['link'].lower()}|{article['published']}"
+            # Create a unique ID based on normalized fields, including description hash
+            title = article["title"].lower().strip()
+            link = article["link"].lower().strip()
+            description = article["description"].strip()
+            published = article["published"]
+            description_hash = hashlib.md5(description.encode('utf-8')).hexdigest()
+            doc_id = f"{title}|{link}|{published}|{description_hash}"
             if doc_id in existing_ids:
                 logger.debug(f"Skipping duplicate in DB: {doc_id}")
                 continue
@@ -156,7 +159,7 @@ def process_and_store_articles(articles):
                 "category": article["category"],
                 "image": article["image"],
             }
-            doc = Document(page_content=article["description"], metadata=metadata, id=doc_id)
+            doc = Document(page_content=description, metadata=metadata, id=doc_id)
             documents.append(doc)
         except Exception as e:
             logger.error(f"Error processing article {article['title']}: {e}")