Update rss_processor.py

rss_processor.py

@@ -80,7 +80,7 @@ def fetch_rss_feeds():
             for entry in feed.entries[:5]:
                 title = entry.get("title", "No Title")
                 link = entry.get("link", "")
-                # Create a unique key for deduplication
+                # Create a unique key for deduplication (title and link)
                 article_key = f"{title}|{link}"
                 if article_key not in seen_articles:
                     seen_articles.add(article_key)
@@ -93,7 +93,7 @@ def fetch_rss_feeds():
                         "category": categorize_feed(feed_url),
                         "image": image if image else "",
                     })
-            logger.info(f"Processed {len(feed.entries[:5])} unique entries from {feed_url}")
+            logger.info(f"Processed {len([e for e in feed.entries[:5] if f'{e.get('title', 'No Title')}|{e.get('link', '')}' not in seen_articles])} unique entries from {feed_url}")
         except Exception as e:
             logger.error(f"Error fetching {feed_url}: {e}")
     return articles
@@ -114,21 +114,25 @@ def categorize_feed(url):
 
 def process_and_store_articles(articles):
     documents = []
+    seen_docs = set()  # Additional de-duplication at DB level
     for article in articles:
         try:
-            metadata = {
-                "title": article["title"] or "No Title",
-                "link": article["link"] or "",
-                "original_description": article["description"] or "No Description",
-                "published": article["published"] or "Unknown Date",
-                "category": article["category"] or "Uncategorized",
-                "image": article["image"] or "",
-            }
-            doc = Document(
-                page_content=article["description"] or "No Description",
-                metadata=metadata
-            )
-            documents.append(doc)
+            key = f"{article['title']}|{article['link']}"
+            if key not in seen_docs:
+                seen_docs.add(key)
+                metadata = {
+                    "title": article["title"] or "No Title",
+                    "link": article["link"] or "",
+                    "original_description": article["description"] or "No Description",
+                    "published": article["published"] or "Unknown Date",
+                    "category": article["category"] or "Uncategorized",
+                    "image": article["image"] or "",
+                }
+                doc = Document(
+                    page_content=article["description"] or "No Description",
+                    metadata=metadata
+                )
+                documents.append(doc)
         except Exception as e:
             logger.error(f"Error processing article {article['title']}: {e}")
     try: