Update app.py

app.py

@@ -1,7 +1,7 @@
 import os
 import threading
 from flask import Flask, render_template, request, jsonify
-from rss_processor import fetch_rss_feeds, process_and_store_articles, vector_db, download_from_hf_hub, upload_to_hf_hub
+from rss_processor import fetch_rss_feeds, process_and_store_articles, vector_db, download_from_hf_hub, upload_to_hf_hub, clean_text
 import logging
 import time
 from datetime import datetime
@@ -61,12 +61,18 @@ def index():
         for doc, meta in zip(all_docs['documents'], all_docs['metadatas']):
             if not meta:
                 continue
-            title = meta.get("title", "No Title").strip().lower()
-            link = meta.get("link", "").strip().lower()
-            description = meta.get("original_description", "No Description").strip()
+            title = meta.get("title", "No Title")
+            link = meta.get("link", "")
+            description = meta.get("original_description", "No Description")
             published = meta.get("published", "Unknown Date").strip()
-            # Use a robust key with normalized fields and description hash for deduplication
-            description_hash = hashlib.md5(description.encode('utf-8')).hexdigest()
+
+            # Clean and normalize all fields
+            title = clean_text(title)
+            link = clean_text(link)
+            description = clean_text(description)
+
+            # Use a robust key with cleaned fields and description hash for deduplication
+            description_hash = hashlib.sha256(description.encode('utf-8')).hexdigest()
             key = f"{title}|{link}|{published}|{description_hash}"
             if key not in seen_keys:
                 seen_keys.add(key)
@@ -130,11 +136,17 @@ def search():
         seen_keys = set()
         for doc in results:
             meta = doc.metadata
-            title = meta.get("title", "No Title").strip().lower()
-            link = meta.get("link", "").strip().lower()
-            description = meta.get("original_description", "No Description").strip()
+            title = meta.get("title", "No Title")
+            link = meta.get("link", "")
+            description = meta.get("original_description", "No Description")
             published = meta.get("published", "Unknown Date").strip()
-            description_hash = hashlib.md5(description.encode('utf-8')).hexdigest()
+
+            # Clean and normalize all fields
+            title = clean_text(title)
+            link = clean_text(link)
+            description = clean_text(description)
+
+            description_hash = hashlib.sha256(description.encode('utf-8')).hexdigest()
             key = f"{title}|{link}|{published}|{description_hash}"
             if key not in seen_keys:
                 seen_keys.add(key)
@@ -182,11 +194,17 @@ def get_updates():
         for doc, meta in zip(all_docs['documents'], all_docs['metadatas']):
             if not meta:
                 continue
-            title = meta.get("title", "No Title").strip().lower()
-            link = meta.get("link", "").strip().lower()
-            description = meta.get("original_description", "No Description").strip()
+            title = meta.get("title", "No Title")
+            link = meta.get("link", "")
+            description = meta.get("original_description", "No Description")
             published = meta.get("published", "Unknown Date").strip()
-            description_hash = hashlib.md5(description.encode('utf-8')).hexdigest()
+
+            # Clean and normalize all fields
+            title = clean_text(title)
+            link = clean_text(link)
+            description = clean_text(description)
+
+            description_hash = hashlib.sha256(description.encode('utf-8')).hexdigest()
             key = f"{title}|{link}|{published}|{description_hash}"
             if key not in seen_keys:
                 seen_keys.add(key)
@@ -218,7 +236,7 @@ def get_updates():
             unique_articles = []
             seen_cat_keys = set()
             for article in sorted(categorized_articles[cat], key=lambda x: x["published"], reverse=True):
-                key = f"{article['title'].lower()}|{article['link'].lower()}|{article['published']}"
+                key = f"{clean_text(article['title'])}|{clean_text(article['link'])}|{article['published']}"
                 if key not in seen_cat_keys:
                     seen_cat_keys.add(key)
                     unique_articles.append(article)
@@ -241,11 +259,17 @@ def get_all_articles(category):
         for doc, meta in zip(all_docs['documents'], all_docs['metadatas']):
             if not meta or meta.get("category") != category:
                 continue
-            title = meta.get("title", "No Title").strip().lower()
-            link = meta.get("link", "").strip().lower()
-            description = meta.get("original_description", "No Description").strip()
+            title = meta.get("title", "No Title")
+            link = meta.get("link", "")
+            description = meta.get("original_description", "No Description")
             published = meta.get("published", "Unknown Date").strip()
-            description_hash = hashlib.md5(description.encode('utf-8')).hexdigest()
+
+            # Clean and normalize all fields
+            title = clean_text(title)
+            link = clean_text(link)
+            description = clean_text(description)
+
+            description_hash = hashlib.sha256(description.encode('utf-8')).hexdigest()
             key = f"{title}|{link}|{published}|{description_hash}"
             if key not in seen_keys:
                 seen_keys.add(key)