Update app.py

app.py

@@ -1,10 +1,7 @@
 import os
-import subprocess
 from flask import Flask, render_template, request, jsonify
 from rss_processor import fetch_rss_feeds, process_and_store_articles, vector_db
 import logging
-import time
-import hashlib
 
 app = Flask(__name__)
 
@@ -12,119 +9,91 @@ app = Flask(__name__)
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-def load_feeds_in_background():
-    logger.info("Starting to fetch and process RSS feeds in background")
-    start_time = time.time()
-    articles = fetch_rss_feeds()
-    logger.info(f"Fetched {len(articles)} articles")
-    process_and_store_articles(articles)
-    logger.info("Articles processed and stored")
-    end_time = time.time()
-    logger.info(f"RSS feed loading took {end_time - start_time:.2f} seconds")
-
 @app.route('/')
 def index():
-    # Show all existing articles immediately, even if empty
     try:
-        # Get all documents from Chroma DB using get()
+        # Fetch and store articles synchronously on first load
+        articles = fetch_rss_feeds()
+        logger.info(f"Fetched {len(articles)} articles")
+        process_and_store_articles(articles)
+        logger.info("Articles processed and stored")
+
+        # Retrieve all articles from Chroma DB
         all_docs = vector_db.get(include=['documents', 'metadatas'])
-        if 'metadatas' in all_docs and all_docs['metadatas']:
-            stored_docs = [
-                Document(page_content=doc['documents'][0] if doc['documents'] else "", metadata=meta)
-                for doc, meta in zip(all_docs['documents'], all_docs['metadatas'])
-            ]
-            logger.info(f"Found {len(stored_docs)} documents in vector DB")
-        else:
-            stored_docs = []
-            logger.warning("No metadata or documents found in vector DB")
-        # Use a set to ensure unique articles by title, link, and full description hash
-        unique_articles = {}
-        for doc in stored_docs:
-            if not doc.metadata:  # Handle potential None metadata
+        if not all_docs.get('metadatas'):
+            logger.warning("No articles in DB yet")
+            return render_template("index.html", categorized_articles={}, has_articles=False)
+
+        # Process retrieved documents
+        enriched_articles = []
+        seen_keys = set()
+        for doc, meta in zip(all_docs['documents'], all_docs['metadatas']):
+            if not meta:
                 continue
-            title = doc.metadata.get("title", "No Title")
-            link = doc.metadata.get("link", "")
-            description = doc.metadata.get("original_description", "No Description")
-            desc_hash = hashlib.md5(description.encode()).hexdigest()
-            key = f"{title}|{link}|{desc_hash}"
-            if key not in unique_articles:
-                unique_articles[key] = {
+            title = meta.get("title", "No Title")
+            link = meta.get("link", "")
+            description = meta.get("original_description", "No Description")
+            key = f"{title}|{link}"
+            if key not in seen_keys:
+                seen_keys.add(key)
+                enriched_articles.append({
                     "title": title,
                     "link": link,
                     "description": description,
-                    "category": doc.metadata.get("category", "Uncategorized"),
-                    "published": doc.metadata.get("published", "Unknown Date"),
-                    "image": doc.metadata.get("image", "svg"),
-                }
-        enriched_articles = list(unique_articles.values())
-        logger.info(f"Enriched {len(enriched_articles)} unique articles for display")
-    except Exception as e:
-        logger.error(f"Error retrieving documents from vector DB: {e}")
-        enriched_articles = []  # Fallback if DB is empty or inaccessible
-
-    # Start loading new feeds in the background
-    subprocess.Popen(["python", "rss_processor.py", "load_feeds"])
+                    "category": meta.get("category", "Uncategorized"),
+                    "published": meta.get("published", "Unknown Date"),
+                    "image": meta.get("image", "svg"),
+                })
+        logger.info(f"Displaying {len(enriched_articles)} unique articles")
 
-    categorized_articles = {}
-    for article in enriched_articles:
-        cat = article["category"]
-        if cat not in categorized_articles:
-            categorized_articles[cat] = []
-        categorized_articles[cat].append(article)
+        # Categorize articles
+        categorized_articles = {}
+        for article in enriched_articles:
+            cat = article["category"]
+            categorized_articles.setdefault(cat, []).append(article)
 
-    return render_template("index.html", categorized_articles=categorized_articles, loading_new_feeds=True, has_articles=bool(enriched_articles))
+        return render_template("index.html", categorized_articles=categorized_articles, has_articles=True)
+    except Exception as e:
+        logger.error(f"Error in index: {e}")
+        return render_template("index.html", categorized_articles={}, has_articles=False)
 
 @app.route('/search', methods=['POST'])
 def search():
     query = request.form.get('search')
-    if query:
-        logger.info(f"Processing search query: {query}")
-        try:
-            results = vector_db.similarity_search(query, k=10)
-            unique_search_articles = {}
-            for doc in results:
-                title = doc.metadata.get("title", "No Title")
-                link = doc.metadata.get("link", "")
-                description = doc.metadata.get("original_description", "No Description")
-                desc_hash = hashlib.md5(description.encode()).hexdigest()
-                key = f"{title}|{link}|{desc_hash}"
-                if key not in unique_search_articles:
-                    unique_search_articles[key] = {
-                        "title": title,
-                        "link": link,
-                        "description": description,
-                        "category": doc.metadata.get("category", "Uncategorized"),
-                        "published": doc.metadata.get("published", "Unknown Date"),
-                        "image": doc.metadata.get("image", "svg"),
-                    }
-            enriched_articles = list(unique_search_articles.values())
-            logger.info(f"Search returned {len(enriched_articles)} unique results")
-        except Exception as e:
-            logger.error(f"Error performing search: {e}")
-            enriched_articles = []
+    if not query:
+        return render_template("index.html", categorized_articles={}, has_articles=False)
+
+    try:
+        logger.info(f"Searching for: {query}")
+        results = vector_db.similarity_search(query, k=10)
+        enriched_articles = []
+        seen_keys = set()
+        for doc in results:
+            meta = doc.metadata
+            title = meta.get("title", "No Title")
+            link = meta.get("link", "")
+            description = meta.get("original_description", "No Description")
+            key = f"{title}|{link}"
+            if key not in seen_keys:
+                seen_keys.add(key)
+                enriched_articles.append({
+                    "title": title,
+                    "link": link,
+                    "description": description,
+                    "category": meta.get("category", "Uncategorized"),
+                    "published": meta.get("published", "Unknown Date"),
+                    "image": meta.get("image", "svg"),
+                })
 
         categorized_articles = {}
         for article in enriched_articles:
             cat = article["category"]
-            if cat not in categorized_articles:
-                categorized_articles[cat] = []
-            categorized_articles[cat].append(article)
-
-        return render_template("index.html", categorized_articles=categorized_articles, loading_new_feeds=True, has_articles=bool(enriched_articles))
-    return render_template("index.html", categorized_articles={}, loading_new_feeds=True, has_articles=False)
+            categorized_articles.setdefault(cat, []).append(article)
 
-@app.route('/check_feeds', methods=['GET'])
-def check_feeds():
-    try:
-        # Check if vector DB has any documents
-        all_docs = vector_db.get(include=['documents', 'metadatas'])
-        if 'metadatas' in all_docs and all_docs['metadatas']:
-            logger.info("Feeds loaded successfully in vector DB")
-            return jsonify({"status": "loaded"})
-        return jsonify({"status": "loading"}), 202
+        return render_template("index.html", categorized_articles=categorized_articles, has_articles=bool(enriched_articles))
     except Exception as e:
-        logger.error(f"Error checking feeds: {e}")
-        return jsonify({"status": "error", "message": str(e)}), 500
+        logger.error(f"Search error: {e}")
+        return render_template("index.html", categorized_articles={}, has_articles=False)
 
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=7860)