text extractor changes

nuse_modules/google_search.py

@@ -4,7 +4,17 @@ import os
 import requests
 import time
 from typing import List
-from boilerpy3 import extractors          # ← switched library
+from boilerpy3 import extractors
+
+article_extractor = extractors.ArticleExtractor()
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/114.0.0.0 Safari/537.36"
+    )
+}
 
 GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
 GOOGLE_CX_ID   = os.getenv("GOOGLE_CX_ID")
@@ -14,12 +24,9 @@ article_extractor = extractors.ArticleExtractor()
 
 
 def extract_full_text(url: str) -> str:
-    """
-    Download a page and return its readable main text.
-    Falls back to empty string on any failure.
-    """
     try:
-        return article_extractor.get_content_from_url(url) or ""
+        html = requests.get(url, headers=HEADERS, timeout=10).text
+        return article_extractor.get_content(html) or ""
     except Exception as e:
         print(f"[SCRAPER ERROR] {url}: {e}")
         return ""