fixing the article generation

nuse_modules/headlines_generator.py

@@ -1,13 +1,17 @@
 from __future__ import annotations
 import datetime as _dt
 import json, os
+import re
 from typing import List, Dict
 
-from clients.redis_client import redis_client as _r              
-from nuse_modules.google_search import search_google_news       
+from clients.redis_client import redis_client as _r
+from nuse_modules.google_search import search_google_news
 from models_initialization.mistral_registry import mistral_generate
 
 
+# ──────────────────────────────────────────────────────────────
+# Config
+# ──────────────────────────────────────────────────────────────
 _CATEGORIES = {
     "world":         "world news top stories",
     "india":         "india top stories",
@@ -18,8 +22,12 @@ _CATEGORIES = {
 
 _ARTICLES_PER_CAT   = 5
 _SUMMARY_TOKENS     = 120
-_REDIS_TTL_SECONDS  = 24 * 3600  
+_REDIS_TTL_SECONDS  = 24 * 3600
 
+
+# ──────────────────────────────────────────────────────────────
+# Helpers
+# ──────────────────────────────────────────────────────────────
 def _dedupe_urls(articles: List[dict]) -> List[dict]:
     seen = set()
     out  = []
@@ -30,19 +38,26 @@ def _dedupe_urls(articles: List[dict]) -> List[dict]:
     return out
 
 
-def _summarise_article(article: dict) -> str:
+def _summarise_article(content: str) -> str:
     prompt = (
         "You are a concise news assistant. Summarise the following article "
         "in one sentence (<=25 words). Omit source and author names.\n\n"
-        f"ARTICLE:\n{article['content']}"
+        f"ARTICLE:\n{content}"
     )
-    return mistral_generate(prompt, max_new_tokens=_SUMMARY_TOKENS, temperature=0.3)
+    raw_output = mistral_generate(prompt, max_new_tokens=_SUMMARY_TOKENS, temperature=0.3)
+
+    # Remove repeated prompt instructions if echoed back
+    cleaned = re.sub(r"(you are.*?article[:\n]+)", "", raw_output, flags=re.IGNORECASE | re.DOTALL).strip()
+    return cleaned
 
 
 def _redis_key(date: str, category: str) -> str:
     return f"headlines:{date}:{category}"
 
 
+# ──────────────────────────────────────────────────────────────
+# Main Generator
+# ──────────────────────────────────────────────────────────────
 def generate_and_store_headlines(today: str | None = None) -> Dict[str, List[dict]]:
     """
     Fetches top articles per category, summarises them, stores in Redis,
@@ -54,26 +69,24 @@ def generate_and_store_headlines(today: str | None = None) -> Dict[str, List[dic
     for cat, query in _CATEGORIES.items():
         print(f"[HEADLINES] {cat.title()} …")
 
-        # 1. Google -> list of {title, link, snippet, content}
         raw_articles = search_google_news([query], num_results=_ARTICLES_PER_CAT)
         raw_articles = _dedupe_urls(raw_articles)
 
-        # 2. Summarise each article
         summaries = []
-        for art in raw_articles:
-            if not art["content"]:
-                continue  # skip if scraper failed
-            summary = _summarise_article(art)
-            summaries.append(
-                {
-                    "title": art["title"],
-                    "url":   art["link"],
-                    "summary": summary,
-                    "source_snippet": art["snippet"],
-                }
-            )
-
-        # 3. Store in Upstash Redis
+        for item in raw_articles:
+            if not item.get("content"):
+                continue  # skip if no full text extracted
+
+            summary = _summarise_article(item["content"])
+
+            summaries.append({
+                "title":   item.get("title"),
+                "url":     item.get("link"),
+                "summary": summary,
+                "source_snippet": item.get("snippet"),
+                "image":   item.get("image"),  # added in google_search.py
+            })
+
         redis_key = _redis_key(date_str, cat)
         _r.set(redis_key, json.dumps(summaries), ex=_REDIS_TTL_SECONDS)