Update app.py

app.py

@@ -1,8 +1,9 @@
 import os
-from flask import Flask, render_template, request, Response
+from flask import Flask, render_template, request, Response, jsonify
 from rss_processor import fetch_rss_feeds, process_and_store_articles, vector_db
 import logging
 import time
+from threading import Thread
 
 app = Flask(__name__)
 
@@ -10,13 +11,8 @@ app = Flask(__name__)
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-@app.route('/')
-def loading():
-    return render_template("loading.html")
-
-@app.route('/load_feeds', methods=['GET'])
-def load_feeds():
-    logger.info("Starting to fetch and process RSS feeds")
+def load_feeds_in_background():
+    logger.info("Starting to fetch and process RSS feeds in background")
     start_time = time.time()
     articles = fetch_rss_feeds()
     logger.info(f"Fetched {len(articles)} articles")
@@ -24,19 +20,50 @@ def load_feeds():
     logger.info("Articles processed and stored")
     end_time = time.time()
     logger.info(f"RSS feed loading took {end_time - start_time:.2f} seconds")
-    return Response("Feeds loaded", status=200)
+
+@app.route('/')
+def loading():
+    # Start loading feeds in a background thread
+    thread = Thread(target=load_feeds_in_background)
+    thread.daemon = True
+    thread.start()
+    return render_template("loading.html")
+
+@app.route('/check_feeds', methods=['GET'])
+def check_feeds():
+    try:
+        # Check if vector DB has documents (simplified check)
+        docs = vector_db.similarity_search("news", k=1)
+        if docs:
+            return jsonify({"status": "loaded"})
+        return jsonify({"status": "loading"}), 202
+    except Exception as e:
+        logger.error(f"Error checking feeds: {e}")
+        return jsonify({"status": "error", "message": str(e)}), 500
 
 @app.route('/index', methods=['GET'])
 def index():
-    stored_docs = vector_db.similarity_search("news", k=1000)  # Increased k to ensure all unique articles
-    # Use a set to ensure unique articles by title and link
+    # Poll until feeds are loaded
+    while True:
+        response = check_feeds()
+        if response.status_code == 200 and response.get_json()["status"] == "loaded":
+            break
+        time.sleep(1)  # Check every second
+
+    stored_docs = vector_db.similarity_search("news", k=1000)  # Increased k for all unique articles
+    # Use a set to ensure unique articles by title, link, and description hash
     unique_articles = {}
     for doc in stored_docs:
-        key = f"{doc.metadata['title']}|{doc.metadata['link']}"
+        import hashlib
+        title = doc.metadata["title"]
+        link = doc.metadata["link"]
+        desc = doc.metadata["original_description"]
+        desc_hash = hashlib.md5(desc.encode()).hexdigest()[:10]  # Short hash for uniqueness
+        key = f"{title}|{link}|{desc_hash}"
         if key not in unique_articles:
             unique_articles[key] = {
-                "title": doc.metadata["title"],
-                "link": doc.metadata["link"],
+                "title": title,
+                "link": link,
                 "description": doc.metadata["original_description"],
                 "category": doc.metadata["category"],
                 "published": doc.metadata["published"],
@@ -52,11 +79,15 @@ def index():
             results = vector_db.similarity_search(query, k=10)
             unique_search_articles = {}
             for doc in results:
-                key = f"{doc.metadata['title']}|{doc.metadata['link']}"
+                title = doc.metadata["title"]
+                link = doc.metadata["link"]
+                desc = doc.metadata["original_description"]
+                desc_hash = hashlib.md5(desc.encode()).hexdigest()[:10]
+                key = f"{title}|{link}|{desc_hash}"
                 if key not in unique_search_articles:
                     unique_search_articles[key] = {
-                        "title": doc.metadata["title"],
-                        "link": doc.metadata["link"],
+                        "title": title,
+                        "link": link,
                         "description": doc.metadata["original_description"],
                         "category": doc.metadata["category"],
                         "published": doc.metadata["published"],