Update rss_processor.py

rss_processor.py

@@ -6,6 +6,7 @@ from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.docstore.document import Document
 import shutil
 import logging
+import hashlib
 
 # Setup logging
 logging.basicConfig(level=logging.INFO)
@@ -69,7 +70,7 @@ vector_db = Chroma(persist_directory=LOCAL_DB_DIR, embedding_function=embedding_
 
 def fetch_rss_feeds():
     articles = []
-    seen_articles = set()  # Track unique articles by title, link, and description
+    seen_articles = set()  # Track unique articles by title, link, and description hash
     for feed_url in RSS_FEEDS:
         try:
             logger.info(f"Fetching feed: {feed_url}")
@@ -82,8 +83,9 @@ def fetch_rss_feeds():
                 title = entry.get("title", "No Title")
                 link = entry.get("link", "")
                 description = entry.get("summary", entry.get("description", "No Description"))
-                # Create a unique key for deduplication (title, link, and description for stricter uniqueness)
-                article_key = f"{title}|{link}|{description[:50]}"  # Use first 50 chars of description to avoid overly long keys
+                # Use MD5 hash of description for uniqueness
+                desc_hash = hashlib.md5(description.encode()).hexdigest()[:10]
+                article_key = f"{title}|{link}|{desc_hash}"
                 if article_key not in seen_articles:
                     seen_articles.add(article_key)
                     unique_count += 1
@@ -120,7 +122,8 @@ def process_and_store_articles(articles):
     seen_docs = set()  # Additional de-duplication at DB level
     for article in articles:
         try:
-            key = f"{article['title']}|{article['link']}|{article['description'][:50]}"
+            desc_hash = hashlib.md5(article["description"].encode()).hexdigest()[:10]
+            key = f"{article['title']}|{article['link']}|{desc_hash}"
             if key not in seen_docs:
                 seen_docs.add(key)
                 metadata = {