Update app.py

app.py

@@ -6,6 +6,9 @@ from urllib.parse import urlparse
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 from trafilatura import fetch_url, extract
+from trafilatura import extract
+from trafilatura.settings import use_config
+from urllib.request import urlopen, Request
 import json
 from huggingface_hub import InferenceClient
 import random
@@ -393,8 +396,16 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="trafilatura",
                             session.headers.update({'User-Agent': ua})
                             content = scrape_with_bs4(url, session)
                         else:  # trafilatura
-                            downloaded = fetch_url(url, headers={'User-Agent': ua})
-                            content = extract(downloaded)
+                            # Use urllib to handle custom headers for trafilatura
+                            req = Request(url, headers={'User-Agent': ua})
+                            with urlopen(req) as response:
+                                downloaded = response.read()
+                            
+                            # Configure trafilatura to use a specific user agent
+                            config = use_config()
+                            config.set("DEFAULT", "USER_AGENT", ua)
+                            
+                            content = extract(downloaded, config=config)
                         
                         if content:
                             break
@@ -404,6 +415,9 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="trafilatura",
                             continue
                         else:
                             raise
+                    except Exception as e:
+                        logger.error(f"Error scraping {url} with User-Agent {ua}: {str(e)}")
+                        continue
                 
                 if not content:
                     logger.warning(f"Failed to scrape content from {url} after trying multiple User-Agents")