headlines revamp

nuse_modules/fetchHeadlines.py

@@ -1,26 +1,20 @@
-from nuse_modules.google_search import search_google_news
-from models_initialization.mistral_registry import mistral_generate
-
-def summarize_headlines_with_mistral(headlines: list[dict]) -> str:
-    prompt = (
-        "Summarize the following news headlines into a short 3-sentence digest. "
-        "Be factual and neutral. Mention the sources.\n\n"
-    )
-    for item in headlines:
-        prompt += f"- {item['title']} – {item.get('link', '')}\n"
-
-    return mistral_generate(prompt, max_new_tokens=200, temperature=0.5)
+# nuse_modules/fetch_headline_articles.py
 
+from nuse_modules.google_search import search_google_news
 
-def generate_headline_digest(keywords: list[str], num_results: int = 5):
-    articles = search_google_news(keywords, num_results=num_results)
+def fetch_headline_articles():
+    queries = [
+        "India news", "US politics", "UK elections", "China economy",
+        "Tech layoffs", "Ukraine war", "AI regulation", "Africa development",
+        "South America inflation", "Global stock markets", "Climate change",
+        "Middle East", "EU summit", "Canada economy", "Australia news",
+        "Russia sanctions", "Elections 2025", "Big tech", "Trade wars",
+        "Global protests", "Public health", "Oil prices", "Space news",
+        "Cryptocurrency", "Cybersecurity"
+    ]
 
-    if isinstance(articles, dict) and "error" in articles:
-        return {"error": articles["error"]}
+    print("[INFO] Fetching news articles from Google Custom Search...")
+    articles = search_google_news(queries, results_per_query=30)  # 30 per query × 25 queries = ~750 raw
 
-    summary = summarize_headlines_with_mistral(articles)
-    return {
-        "summary": summary,
-        "sources": list({article["link"] for article in articles}),
-        "headlines": [article["title"] for article in articles]
-    }
+    print(f"[INFO] Retrieved {len(articles)} unique articles.")
+    return articles

nuse_modules/google_search.py

@@ -2,10 +2,58 @@
 
 import os
 import requests
+import time
+from typing import List
 
 GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
 GOOGLE_CX_ID = os.getenv("GOOGLE_CX_ID")
 
+def search_google_news_batch(queries: List[str], results_per_query: int = 30) -> List[dict]:
+    all_results = []
+    seen_links = set()
+
+    for query in queries:
+        print(f"[SEARCH] Query: {query}")
+        total_fetched = 0
+        start_index = 1
+
+        while total_fetched < results_per_query and start_index <= 91:
+            url = (
+                f"https://www.googleapis.com/customsearch/v1"
+                f"?key={GOOGLE_API_KEY}&cx={GOOGLE_CX_ID}"
+                f"&q={query}&num=10&start={start_index}"
+            )
+
+            try:
+                res = requests.get(url, timeout=10)
+                res.raise_for_status()
+                data = res.json()
+                items = data.get("items", [])
+
+                if not items:
+                    break  # No more results
+
+                for item in items:
+                    link = item.get("link")
+                    if link and link not in seen_links:
+                        seen_links.add(link)
+                        all_results.append({
+                            "title": item.get("title"),
+                            "link": link,
+                            "snippet": item.get("snippet"),
+                            "query": query,
+                        })
+
+                total_fetched += len(items)
+                start_index += 10
+                time.sleep(0.5)  # Avoid rate limits
+
+            except Exception as e:
+                print(f"[ERROR] Query '{query}' failed at start={start_index}: {e}")
+                break
+
+    return all_results
+
 def search_google_news(keywords: list[str], num_results: int = 5):
     query = " ".join(keywords)
     url = (

routes/headlines.py

@@ -1,10 +1,9 @@
-from fastapi import APIRouter, Query
-from nuse_modules.fetchHeadlines import generate_headline_digest
+from fastapi import APIRouter
+from nuse_modules.fetchHeadlines import fetch_headline_articles
 
 headlines = APIRouter()
 
 @headlines.get("/headlines")
-def get_headlines(q: str = Query("India", description="Comma-separated keywords to search")):
-    keywords = [kw.strip() for kw in q.split(",")]
-    result = generate_headline_digest(keywords)
-    return result
+def get_headlines():
+    articles = fetch_headline_articles()
+    return {"total": len(articles), "articles": articles}