Update app.py

app.py

@@ -8,6 +8,7 @@ import time
 import random
 import base64
 from io import BytesIO
+from googlesearch import search
 
 app = Flask(__name__)
 
@@ -127,33 +128,8 @@ def scrape_site_content(query, num_sites=5):
     scraped = 0
 
     try:
-        # Use a more direct search URL format
-        search_url = f"https://www.bing.com/search?q={urllib.parse.quote(query)}&num={num_sites}"
-        search_response = requests.get(search_url, headers=headers, timeout=30)
-        search_response.raise_for_status()
-
-        # Parse the search results
-        search_soup = BeautifulSoup(search_response.text, 'html.parser')
-
-        # Look for URLs in multiple possible locations
-        search_results = []
-
-        # Method 1: Look for cite elements
-        for cite in search_soup.find_all('cite'):
-            url = cite.text.strip()
-            if url.startswith(('http://', 'https://')):
-                search_results.append(url)
-
-        # Method 2: Look for links with specific attributes
-        for a in search_soup.find_all('a'):
-            href = a.get('href', '')
-            if 'url?q=' in href:
-                url = href.split('url?q=')[1].split('&')[0]
-                if url.startswith(('http://', 'https://')):
-                    search_results.append(urllib.parse.unquote(url))
-
-        # Remove duplicates while preserving order
-        search_results = list(dict.fromkeys(search_results))
+        # Use googlesearch-python to get URLs
+        search_results = search(query, num_results=num_sites)
 
         # Process each found URL
         for url in search_results: