Update app.py

app.py

@@ -541,12 +541,22 @@ async def website_summarizer(url: str, proxy: Optional[str] = None):
 @app.get("/api/ask_website")
 async def ask_website(url: str, question: str, model: str = "llama-3-70b", proxy: Optional[str] = None):
     """
-    Asks a question about the content of a given website.
+    Asks a question about the content of a given website using Lynx user agent for better compatibility.
     """
     try:
-        # Extract text from the given URL
+        # Generate random Lynx user agent
+        def get_useragent():
+            lynx_version = f"Lynx/{random.randint(2, 3)}.{random.randint(8, 9)}.{random.randint(0, 2)}"
+            libwww_version = f"libwww-FM/{random.randint(2, 3)}.{random.randint(13, 15)}"
+            ssl_mm_version = f"SSL-MM/{random.randint(1, 2)}.{random.randint(3, 5)}"
+            openssl_version = f"OpenSSL/{random.randint(1, 3)}.{random.randint(0, 4)}.{random.randint(0, 9)}"
+            return f"{lynx_version} {libwww_version} {ssl_mm_version} {openssl_version}"
+
+        # Extract text from the given URL using Lynx user agent
         proxies = {'http': proxy, 'https': proxy} if proxy else None
-        response = requests.get(url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/111.0"}, proxies=proxies)
+        headers = {"User-Agent": get_useragent()}
+        
+        response = requests.get(url, headers=headers, proxies=proxies)
         response.raise_for_status()
         visible_text = extract_text_from_webpage(response.text)
         if len(visible_text) > 7500:  # Adjust max_chars based on your needs