Update app.py

app.py

@@ -17,36 +17,41 @@ def index():
     process_and_store_articles(articles)
     logger.info("Articles processed and stored")
     stored_docs = vector_db.similarity_search("news", k=len(articles))
-    enriched_articles = [
-        {
-            "title": doc.metadata["title"],
-            "link": doc.metadata["link"],
-            "description": doc.metadata["original_description"],
-            "category": doc.metadata["category"],
-            "published": doc.metadata["published"],
-            "image": doc.metadata.get("image", "svg"),
-        }
-        for doc in stored_docs
-    ]
-    logger.info(f"Enriched {len(enriched_articles)} articles for display")
+    # Use a set to ensure unique articles by title and link
+    unique_articles = {}
+    for doc in stored_docs:
+        key = f"{doc.metadata['title']}|{doc.metadata['link']}"
+        if key not in unique_articles:
+            unique_articles[key] = {
+                "title": doc.metadata["title"],
+                "link": doc.metadata["link"],
+                "description": doc.metadata["original_description"],
+                "category": doc.metadata["category"],
+                "published": doc.metadata["published"],
+                "image": doc.metadata.get("image", "svg"),
+            }
+    enriched_articles = list(unique_articles.values())
+    logger.info(f"Enriched {len(enriched_articles)} unique articles for display")
 
     if request.method == 'POST' and 'search' in request.form:
         query = request.form.get('search')
         if query:
             logger.info(f"Processing search query: {query}")
             results = vector_db.similarity_search(query, k=10)
-            enriched_articles = [
-                {
-                    "title": doc.metadata["title"],
-                    "link": doc.metadata["link"],
-                    "description": doc.metadata["original_description"],
-                    "category": doc.metadata["category"],
-                    "published": doc.metadata["published"],
-                    "image": doc.metadata.get("image", "svg"),
-                }
-                for doc in results
-            ]
-            logger.info(f"Search returned {len(enriched_articles)} results")
+            unique_search_articles = {}
+            for doc in results:
+                key = f"{doc.metadata['title']}|{doc.metadata['link']}"
+                if key not in unique_search_articles:
+                    unique_search_articles[key] = {
+                        "title": doc.metadata["title"],
+                        "link": doc.metadata["link"],
+                        "description": doc.metadata["original_description"],
+                        "category": doc.metadata["category"],
+                        "published": doc.metadata["published"],
+                        "image": doc.metadata.get("image", "svg"),
+                    }
+            enriched_articles = list(unique_search_articles.values())
+            logger.info(f"Search returned {len(enriched_articles)} unique results")
 
     categorized_articles = {}
     for article in enriched_articles: