Update application/utils/web_search.py

application/utils/web_search.py

@@ -1,51 +1,43 @@
 import requests
-from googlesearch import search
 from bs4 import BeautifulSoup
 import re
-import random
+from duckduckgo_search import DDGS
+
 class WebScarper:
     def __init__(self):
-        pass
-    def get_url(self,query):
-        results = []
-        for result in search(query, num_results=3):
-            results.append(result)
-        if(results!=[]):
-            return random.choice(results)
-        else:
-            return None
+        self.ddgs = DDGS()
+
+    def get_urls(self, query):
+        results = self.ddgs.text(query, max_results=3)
+        return [result['href'] for result in results] if results else []
+
     def fetch_url(self, url):
-        try: 
-            headers = { 
+        try:
+            headers = {
                 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
                 'Accept-Language': 'en-US,en;q=0.9',
                 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
-            } 
-
-            response = requests.get(url, headers=headers)
-            if response.status_code != 200:
-                raise Exception(f"Unable to fetch URL, status code: {response.status_code}")
+            }
+            response = requests.get(url, headers=headers, timeout=10)
+            response.raise_for_status()
             return response.text
-        
-        except Exception as e:
-            print(f"Error: {e}")
+        except requests.exceptions.RequestException as e:
+            print(f"Error fetching URL {url}: {e}")
             return None
 
-
     def get_text(self, data):
         soup = BeautifulSoup(data, 'html.parser')
-        text = soup.get_text() 
-        cleaned_text = re.sub(r'\s+', ' ', text).strip() 
-        if(len(cleaned_text)>4000):
-            return cleaned_text[:4000]
-        else:
-            return cleaned_text
-        
+        text = soup.get_text()
+        cleaned_text = re.sub(r'\s+', ' ', text).strip()
+        return cleaned_text[:4000] if len(cleaned_text) > 4000 else cleaned_text
 
-    def scarpe(self,query):
-        url = self.get_url(query)
-        data = self.fetch_url(url)
-        if(data==None or url==None):
+    def scarpe(self, query):
+        urls = self.get_urls(query)
+        if not urls:
             return None
-        return self.get_text(data)
-    
+
+        for url in urls:
+            data = self.fetch_url(url)
+            if data:
+                return self.get_text(data)
+        return None