Shreyas094
commited on
Commit
•
84b4903
1
Parent(s):
827f459
Update app.py
Browse files
app.py
CHANGED
@@ -88,7 +88,7 @@ def get_random_user_agent(include_searx=False):
|
|
88 |
else:
|
89 |
return random.choice(user_agents)
|
90 |
|
91 |
-
@retry(stop=stop_after_attempt(
|
92 |
def scrape_with_bs4(url, session):
|
93 |
try:
|
94 |
headers = {'User-Agent': get_random_user_agent()}
|
@@ -120,7 +120,7 @@ def scrape_with_bs4(url, session):
|
|
120 |
logger.error(f"Unexpected error scraping {url}: {e}")
|
121 |
return {'success': False, 'error': str(e)}
|
122 |
|
123 |
-
@retry(stop=stop_after_attempt(
|
124 |
def scrape_with_trafilatura(url):
|
125 |
try:
|
126 |
downloaded = fetch_url(url, timeout=10)
|
|
|
88 |
else:
|
89 |
return random.choice(user_agents)
|
90 |
|
91 |
+
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
|
92 |
def scrape_with_bs4(url, session):
|
93 |
try:
|
94 |
headers = {'User-Agent': get_random_user_agent()}
|
|
|
120 |
logger.error(f"Unexpected error scraping {url}: {e}")
|
121 |
return {'success': False, 'error': str(e)}
|
122 |
|
123 |
+
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
|
124 |
def scrape_with_trafilatura(url):
|
125 |
try:
|
126 |
downloaded = fetch_url(url, timeout=10)
|