few more revamps to headlines generation

nuse_modules/headlines_generator.py

@@ -37,11 +37,22 @@ def _dedupe_urls(articles: List[dict]) -> List[dict]:
             out.append(art)
     return out
 
+def is_probably_article(url: str) -> bool:
+    """
+    Simple heuristic: filters out category pages, homepages, etc.
+    """
+    bad_patterns = [
+        "/world", "/us", "/news", "/topics", "/home", "/video",
+        "index.html", ".com/", ".org/", ".net/"
+    ]
+    return not any(url.rstrip("/").endswith(p.strip("/")) for p in bad_patterns)
+
 
 def _summarise_article(content: str) -> str:
     prompt = (
         "You are a concise news assistant. Summarise the following article "
         "in one sentence (<=25 words). Omit source and author names.\n\n"
+        "Some of these articles contain text which is not useful to the context so you can omit it."
         f"ARTICLE:\n{content}"
     )
     raw_output = mistral_generate(prompt, max_new_tokens=_SUMMARY_TOKENS, temperature=0.3)
@@ -74,8 +85,14 @@ def generate_and_store_headlines(today: str | None = None) -> Dict[str, List[dic
 
         summaries = []
         for item in raw_articles:
+
+            link = item.get("link")
+
             if not item.get("content"):
-                continue  # skip if no full text extracted
+                continue  
+
+            if not is_probably_article(link):
+                continue
 
             summary = _summarise_article(item["content"])