Update rss_processor.py

rss_processor.py

@@ -71,7 +71,7 @@ vector_db = Chroma(persist_directory=LOCAL_DB_DIR, embedding_function=embedding_
 
 def fetch_rss_feeds():
     articles = []
-    seen_articles = set()  # Track unique articles by title, link, and description hash
+    seen_articles = set()  # Track unique articles by title, link, and full description hash
     for feed_url in RSS_FEEDS:
         try:
             logger.info(f"Fetching feed: {feed_url}")
@@ -84,8 +84,8 @@ def fetch_rss_feeds():
                 title = entry.get("title", "No Title")
                 link = entry.get("link", "")
                 description = entry.get("summary", entry.get("description", "No Description"))
-                # Use MD5 hash of description for uniqueness
-                desc_hash = hashlib.md5(description.encode()).hexdigest()[:10]
+                # Use full MD5 hash of description for stricter uniqueness
+                desc_hash = hashlib.md5(description.encode()).hexdigest()
                 article_key = f"{title}|{link}|{desc_hash}"
                 if article_key not in seen_articles:
                     seen_articles.add(article_key)
@@ -123,7 +123,7 @@ def process_and_store_articles(articles):
     seen_docs = set()  # Additional de-duplication at DB level
     for article in articles:
         try:
-            desc_hash = hashlib.md5(article["description"].encode()).hexdigest()[:10]
+            desc_hash = hashlib.md5(article["description"].encode()).hexdigest()
             key = f"{article['title']}|{article['link']}|{desc_hash}"
             if key not in seen_docs:
                 seen_docs.add(key)