Spaces:
Running
Running
File size: 1,623 Bytes
bbeca57 a8a618d bbeca57 a8a618d bbeca57 a8a618d bbeca57 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 |
import requests
from googlesearch import search
from bs4 import BeautifulSoup
import re
import random
class WebScarper:
def __init__(self):
pass
def get_url(self,query):
results = []
for result in search(query, num_results=3):
results.append(result)
if(results!=[]):
return random.choice(results)
else:
return None
def fetch_url(self, url):
try:
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
'Accept-Language': 'en-US,en;q=0.9',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
}
response = requests.get(url, headers=headers)
if response.status_code != 200:
raise Exception(f"Unable to fetch URL, status code: {response.status_code}")
return response.text
except Exception as e:
print(f"Error: {e}")
return None
def get_text(self, data):
soup = BeautifulSoup(data, 'html.parser')
text = soup.get_text()
cleaned_text = re.sub(r'\s+', ' ', text).strip()
if(len(cleaned_text)>4000):
return cleaned_text[:4000]
else:
return cleaned_text
def scarpe(self,query):
url = self.get_url(query)
data = self.fetch_url(url)
if(data==None or url==None):
return None
return self.get_text(data)
|