File size: 4,120 Bytes
1be3350 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 |
import time
import logging
import random
import threading
from typing import Optional, Dict, Any
from duckduckgo_search.exceptions import RatelimitException
logger = logging.getLogger(__name__)
class RateLimitedSearch:
"""Rate limited search implementation with exponential backoff."""
def __init__(self):
self.last_request_time = 0
self.min_delay = 30 # Increased minimum delay between requests to 30 seconds
self.max_delay = 300 # Maximum delay of 5 minutes
self.jitter = 5 # Added more jitter range
self.consecutive_failures = 0
self.max_consecutive_failures = 5 # Increased max failures before giving up
self._delay_lock = threading.Lock() # Add thread safety
def _add_jitter(self, delay: float) -> float:
"""Add randomized jitter to delay."""
return delay + random.uniform(-self.jitter, self.jitter)
def _wait_for_rate_limit(self):
"""Wait for rate limit with exponential backoff."""
with self._delay_lock:
current_time = time.time()
elapsed = current_time - self.last_request_time
# Calculate delay based on consecutive failures
if self.consecutive_failures > 0:
delay = min(
self.max_delay,
self.min_delay * (2 ** (self.consecutive_failures - 1))
)
else:
delay = self.min_delay
# Add jitter to prevent synchronized requests
jitter = random.uniform(-self.jitter, self.jitter)
delay = max(0, delay + jitter)
# If not enough time has elapsed, wait
if elapsed < delay:
time.sleep(delay - elapsed)
self.last_request_time = time.time()
def execute_with_retry(self,
search_func: callable,
max_retries: int = 3,
**kwargs) -> Optional[Dict[str, Any]]:
"""Execute search with retries and exponential backoff."""
for attempt in range(max_retries):
try:
# Enforce rate limiting
self._wait_for_rate_limit()
# Execute search
result = search_func(**kwargs)
# Reset consecutive failures on success
self.consecutive_failures = 0
return result
except RatelimitException as e:
self.consecutive_failures += 1
# Calculate backoff time
backoff = min(
self.max_delay,
self.min_delay * (2 ** attempt)
)
backoff = self._add_jitter(backoff)
if attempt == max_retries - 1:
logger.error(f"Rate limit exceeded after {max_retries} retries")
raise
logger.warning(f"Rate limit hit, attempt {attempt + 1}/{max_retries}. "
f"Waiting {backoff:.2f} seconds...")
time.sleep(backoff)
# If we've hit too many consecutive failures, raise an exception
if self.consecutive_failures >= self.max_consecutive_failures:
logger.error("Too many consecutive rate limit failures")
raise RatelimitException("Persistent rate limiting detected")
continue
except Exception as e:
logger.error(f"Search error on attempt {attempt + 1}: {str(e)}")
if attempt == max_retries - 1:
raise
backoff = self.min_delay * (2 ** attempt)
backoff = self._add_jitter(backoff)
logger.info(f"Retrying in {backoff:.2f} seconds...")
time.sleep(backoff)
return None
|