Shreyas094 commited on
Commit
84b4903
1 Parent(s): 827f459

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -88,7 +88,7 @@ def get_random_user_agent(include_searx=False):
88
  else:
89
  return random.choice(user_agents)
90
 
91
- @retry(stop=stop_after_attempt(1), wait=wait_exponential(multiplier=1, min=4, max=10))
92
  def scrape_with_bs4(url, session):
93
  try:
94
  headers = {'User-Agent': get_random_user_agent()}
@@ -120,7 +120,7 @@ def scrape_with_bs4(url, session):
120
  logger.error(f"Unexpected error scraping {url}: {e}")
121
  return {'success': False, 'error': str(e)}
122
 
123
- @retry(stop=stop_after_attempt(1), wait=wait_exponential(multiplier=1, min=4, max=10))
124
  def scrape_with_trafilatura(url):
125
  try:
126
  downloaded = fetch_url(url, timeout=10)
 
88
  else:
89
  return random.choice(user_agents)
90
 
91
+ @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
92
  def scrape_with_bs4(url, session):
93
  try:
94
  headers = {'User-Agent': get_random_user_agent()}
 
120
  logger.error(f"Unexpected error scraping {url}: {e}")
121
  return {'success': False, 'error': str(e)}
122
 
123
+ @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
124
  def scrape_with_trafilatura(url):
125
  try:
126
  downloaded = fetch_url(url, timeout=10)