Update app.py

app.py

@@ -5,6 +5,7 @@ from rss_processor import fetch_rss_feeds, process_and_store_articles, vector_db
 import logging
 import time
 from datetime import datetime
+import hashlib
 
 app = Flask(__name__)
 
@@ -23,9 +24,8 @@ def load_feeds_in_background():
         articles = fetch_rss_feeds()
         logger.info(f"Fetched {len(articles)} articles")
         process_and_store_articles(articles)
-        last_update_time = time.time()  # Update timestamp when new articles are added
+        last_update_time = time.time()
         logger.info("Background feed processing complete")
-        # Upload updated DB to Hugging Face Hub
         upload_to_hf_hub()
         loading_complete = True
     except Exception as e:
@@ -61,11 +61,13 @@ def index():
         for doc, meta in zip(all_docs['documents'], all_docs['metadatas']):
             if not meta:
                 continue
-            title = meta.get("title", "No Title").strip()
-            link = meta.get("link", "").strip()
+            title = meta.get("title", "No Title").strip().lower()
+            link = meta.get("link", "").strip().lower()
+            description = meta.get("original_description", "No Description").strip()
             published = meta.get("published", "Unknown Date").strip()
-            # Use a more robust key with normalized fields to prevent duplicates
-            key = f"{title.lower()}|{link.lower()}|{published}"
+            # Use a robust key with normalized fields and description hash for deduplication
+            description_hash = hashlib.md5(description.encode('utf-8')).hexdigest()
+            key = f"{title}|{link}|{published}|{description_hash}"
             if key not in seen_keys:
                 seen_keys.add(key)
                 try:
@@ -75,13 +77,13 @@ def index():
                 enriched_articles.append({
                     "title": title,
                     "link": link,
-                    "description": meta.get("original_description", "No Description"),
+                    "description": description,
                     "category": meta.get("category", "Uncategorized"),
                     "published": published,
                     "image": meta.get("image", "svg"),
                 })
             else:
-                logger.debug(f"Duplicate found in DB: {key}")
+                logger.debug(f"Duplicate found in retrieval: {key}")
 
         # Sort by published date (stable sort)
         enriched_articles.sort(key=lambda x: x["published"], reverse=True)
@@ -100,7 +102,6 @@ def index():
         # Limit to 10 most recent per category and log top 2 for debugging
         for cat in categorized_articles:
             categorized_articles[cat] = sorted(categorized_articles[cat], key=lambda x: x["published"], reverse=True)[:10]
-            # Log the first two items to check for duplicates
             if len(categorized_articles[cat]) >= 2:
                 logger.debug(f"Category {cat} top 2: {categorized_articles[cat][0]['title']} | {categorized_articles[cat][1]['title']}")
 
@@ -129,16 +130,18 @@ def search():
         seen_keys = set()
         for doc in results:
             meta = doc.metadata
-            title = meta.get("title", "No Title").strip()
-            link = meta.get("link", "").strip()
+            title = meta.get("title", "No Title").strip().lower()
+            link = meta.get("link", "").strip().lower()
+            description = meta.get("original_description", "No Description").strip()
             published = meta.get("published", "Unknown Date").strip()
-            key = f"{title}|{link}|{published}"
+            description_hash = hashlib.md5(description.encode('utf-8')).hexdigest()
+            key = f"{title}|{link}|{published}|{description_hash}"
             if key not in seen_keys:
                 seen_keys.add(key)
                 enriched_articles.append({
                     "title": title,
                     "link": link,
-                    "description": meta.get("original_description", "No Description"),
+                    "description": description,
                     "category": meta.get("category", "Uncategorized"),
                     "published": published,
                     "image": meta.get("image", "svg"),
@@ -179,20 +182,22 @@ def get_updates():
         for doc, meta in zip(all_docs['documents'], all_docs['metadatas']):
             if not meta:
                 continue
-            title = meta.get("title", "No Title").strip()
-            link = meta.get("link", "").strip()
+            title = meta.get("title", "No Title").strip().lower()
+            link = meta.get("link", "").strip().lower()
+            description = meta.get("original_description", "No Description").strip()
             published = meta.get("published", "Unknown Date").strip()
-            key = f"{title}|{link}|{published}"
+            description_hash = hashlib.md5(description.encode('utf-8')).hexdigest()
+            key = f"{title}|{link}|{published}|{description_hash}"
             if key not in seen_keys:
                 seen_keys.add(key)
                 try:
                     published = datetime.strptime(published, "%Y-%m-%d %H:%M:%S").isoformat() if "Unknown" not in published else published
                 except (ValueError, TypeError):
-                    published = "1970-01-01T00:00:00"  # Fallback to a very old date
+                    published = "1970-01-01T00:00:00"
                 enriched_articles.append({
                     "title": title,
                     "link": link,
-                    "description": meta.get("original_description", "No Description"),
+                    "description": description,
                     "category": meta.get("category", "Uncategorized"),
                     "published": published,
                     "image": meta.get("image", "svg"),
@@ -204,7 +209,6 @@ def get_updates():
             cat = article["category"]
             if cat not in categorized_articles:
                 categorized_articles[cat] = []
-            # Extra deduplication for category
             key = f"{article['title']}|{article['link']}|{article['published']}"
             if key not in [f"{a['title']}|{a['link']}|{a['published']}" for a in categorized_articles[cat]]:
                 categorized_articles[cat].append(article)
@@ -214,7 +218,7 @@ def get_updates():
             unique_articles = []
             seen_cat_keys = set()
             for article in sorted(categorized_articles[cat], key=lambda x: x["published"], reverse=True):
-                key = f"{article['title']}|{article['link']}|{article['published']}"
+                key = f"{article['title'].lower()}|{article['link'].lower()}|{article['published']}"
                 if key not in seen_cat_keys:
                     seen_cat_keys.add(key)
                     unique_articles.append(article)
@@ -237,20 +241,22 @@ def get_all_articles(category):
         for doc, meta in zip(all_docs['documents'], all_docs['metadatas']):
             if not meta or meta.get("category") != category:
                 continue
-            title = meta.get("title", "No Title").strip()
-            link = meta.get("link", "").strip()
+            title = meta.get("title", "No Title").strip().lower()
+            link = meta.get("link", "").strip().lower()
+            description = meta.get("original_description", "No Description").strip()
             published = meta.get("published", "Unknown Date").strip()
-            key = f"{title}|{link}|{published}"
+            description_hash = hashlib.md5(description.encode('utf-8')).hexdigest()
+            key = f"{title}|{link}|{published}|{description_hash}"
             if key not in seen_keys:
                 seen_keys.add(key)
                 try:
                     published = datetime.strptime(published, "%Y-%m-%d %H:%M:%S").isoformat() if "Unknown" not in published else published
                 except (ValueError, TypeError):
-                    published = "1970-01-01T00:00:00"  # Fallback to a very old date
+                    published = "1970-01-01T00:00:00"
                 enriched_articles.append({
                     "title": title,
                     "link": link,
-                    "description": meta.get("original_description", "No Description"),
+                    "description": description,
                     "category": meta.get("category", "Uncategorized"),
                     "published": published,
                     "image": meta.get("image", "svg"),