again

components/generators/daily_feed.py

@@ -1,10 +1,12 @@
 import os
 import json
 import redis
+import numpy as np
 from typing import List, Dict
 from openai import OpenAI
 from components.indexers.news_indexer import get_upstash_vector_store
-from llama_index.core import VectorStoreIndex, StorageContext
+from llama_index.core import StorageContext
+from llama_index.vector_stores.types import VectorStoreQuery
 
 # 🔐 Environment variables
 REDIS_URL = os.environ.get("UPSTASH_REDIS_URL", "redis://localhost:6379")
@@ -28,26 +30,29 @@ BASE_PROMPT = (
     "Return up to 3 punchy headlines, each under 20 words, written like a premium editorial bulletin."
 )
 
-# 📥 Load topic-wise documents from Upstash vector store
+# 📥 Load documents grouped by topic from Upstash
 def load_all_documents_grouped_by_topic() -> Dict[str, List[str]]:
-    topic_docs = {}
+    topic_docs = {key: [] for key in TOPIC_KEYS}
 
     try:
         vector_store = get_upstash_vector_store()
-        storage_context = StorageContext.from_defaults(vector_store=vector_store)
 
-        # 🔍 Load all documents from vector store
-        all_nodes = vector_store._data.values()  # Upstash uses `_data` internally to store nodes
-
-        for node in all_nodes:
+        for topic, key in zip(TOPICS, TOPIC_KEYS):
             try:
-                content = node.get_content().strip()
-                topic = node.metadata.get("topic", "").lower().replace(" news", "")
-                if not content or not topic:
-                    continue
-                topic_docs.setdefault(topic, []).append(content)
+                dummy_vector = np.random.rand(384).tolist()  # Assuming MiniLM embeddings
+                query = VectorStoreQuery(
+                    query_embedding=dummy_vector,
+                    similarity_top_k=50,
+                    filters={"topic": topic}
+                )
+                result = vector_store.query(query)
+
+                for node in result.nodes:
+                    content = node.get_content().strip()
+                    if content:
+                        topic_docs[key].append(content)
             except Exception as e:
-                print(f"❌ [Node processing error]", e)
+                print(f"❌ [Topic Metadata Filter error: {key}]", e)
 
     except Exception as e:
         print("❌ [load_all_documents_grouped_by_topic Error]", e)
@@ -62,9 +67,9 @@ def summarize_topic(topic_key: str, docs: List[str]) -> List[Dict]:
 
     try:
         client = OpenAI(api_key=OPENAI_API_KEY)
-        content = "\n\n---\n\n".join(docs)[:12000]  # trim to avoid token overflow
+        content = "\n\n---\n\n".join(docs)[:12000]
 
-        print(f"🧠 Summarizing topic via the OpenAI: {topic_key}")
+        print(f"🧠 Summarizing topic via OpenAI: {topic_key}")
         completion = client.chat.completions.create(
             model="gpt-4",
             messages=[
@@ -96,7 +101,6 @@ def generate_and_cache_daily_feed():
     try:
         print("🆕 Running OpenAI-powered daily feed generator....")
         topic_docs = load_all_documents_grouped_by_topic()
-        print('Loaded topics', topic_docs)
         feed_map = {}
 
         for topic_key in TOPIC_KEYS: