search_utils.py

import time
import logging
import random
import threading
from typing import Optional, Dict, Any
from duckduckgo_search.exceptions import RatelimitException

logger = logging.getLogger(__name__)

class RateLimitedSearch:
    """Rate limited search implementation with exponential backoff."""
    
    def __init__(self):
        self.last_request_time = 0
        self.min_delay = 30  # Increased minimum delay between requests to 30 seconds
        self.max_delay = 300  # Maximum delay of 5 minutes
        self.jitter = 5  # Added more jitter range
        self.consecutive_failures = 0
        self.max_consecutive_failures = 5  # Increased max failures before giving up
        self._delay_lock = threading.Lock()  # Add thread safety
        
    def _add_jitter(self, delay: float) -> float:
        """Add randomized jitter to delay."""
        return delay + random.uniform(-self.jitter, self.jitter)
    
    def _wait_for_rate_limit(self):
        """Wait for rate limit with exponential backoff."""
        with self._delay_lock:
            current_time = time.time()
            elapsed = current_time - self.last_request_time
            
            # Calculate delay based on consecutive failures
            if self.consecutive_failures > 0:
                delay = min(
                    self.max_delay,
                    self.min_delay * (2 ** (self.consecutive_failures - 1))
                )
            else:
                delay = self.min_delay
                
            # Add jitter to prevent synchronized requests
            jitter = random.uniform(-self.jitter, self.jitter)
            delay = max(0, delay + jitter)
            
            # If not enough time has elapsed, wait
            if elapsed < delay:
                time.sleep(delay - elapsed)
                
            self.last_request_time = time.time()
            
    def execute_with_retry(self, 
                          search_func: callable, 
                          max_retries: int = 3,
                          **kwargs) -> Optional[Dict[str, Any]]:
        """Execute search with retries and exponential backoff."""
        
        for attempt in range(max_retries):
            try:
                # Enforce rate limiting
                self._wait_for_rate_limit()
                
                # Execute search
                result = search_func(**kwargs)
                
                # Reset consecutive failures on success
                self.consecutive_failures = 0
                return result
                
            except RatelimitException as e:
                self.consecutive_failures += 1
                
                # Calculate backoff time
                backoff = min(
                    self.max_delay,
                    self.min_delay * (2 ** attempt)
                )
                backoff = self._add_jitter(backoff)
                
                if attempt == max_retries - 1:
                    logger.error(f"Rate limit exceeded after {max_retries} retries")
                    raise
                    
                logger.warning(f"Rate limit hit, attempt {attempt + 1}/{max_retries}. "
                             f"Waiting {backoff:.2f} seconds...")
                time.sleep(backoff)
                
                # If we've hit too many consecutive failures, raise an exception
                if self.consecutive_failures >= self.max_consecutive_failures:
                    logger.error("Too many consecutive rate limit failures")
                    raise RatelimitException("Persistent rate limiting detected")
                continue
                
            except Exception as e:
                logger.error(f"Search error on attempt {attempt + 1}: {str(e)}")
                if attempt == max_retries - 1:
                    raise
                    
                backoff = self.min_delay * (2 ** attempt)
                backoff = self._add_jitter(backoff)
                logger.info(f"Retrying in {backoff:.2f} seconds...")
                time.sleep(backoff)
                
        return None