Update app.py

app.py

@@ -475,9 +475,9 @@ Your response should be detailed, informative, accurate, and directly relevant t
         return "Error: Unable to generate a summary. Please try again."
 
 def search_and_scrape(query, chat_history, num_results=5, scraper="bs4", max_chars=3000, time_range="", language="all", category="", 
-                      engines=[], safesearch=2, method="GET", llm_temperature=0.2, timeout=5):
+                      engines=[], safesearch=2, method="GET", llm_temperature=0.2, timeout=5, use_duckduckgo=False):
     
-     try:
+    try:
         # Step 1: Rephrase the Query
         rephrased_query = rephrase_query(chat_history, query, temperature=llm_temperature)
         logger.info(f"Rephrased Query: {rephrased_query}")
@@ -489,74 +489,76 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="bs4", max_cha
         # Step 2: Perform search
         if use_duckduckgo:
             search_results = duckduckgo_search(rephrased_query, num_results, time_range, language, safesearch)
+            results = search_results  # Assign DuckDuckGo results directly
         else:
-        # Search query parameters
-        params = {
-            'q': rephrased_query,
-            'format': 'json',
-            'time_range': time_range,
-            'language': language,
-            'category': category,
-            'engines': ','.join(engines),
-            'safesearch': safesearch
-        }
+            # Search query parameters
+            params = {
+                'q': rephrased_query,
+                'format': 'json',
+                'time_range': time_range,
+                'language': language,
+                'category': category,
+                'engines': ','.join(engines),
+                'safesearch': safesearch
+            }
+    
+            # Remove empty parameters
+            params = {k: v for k, v in params.items() if v != ""}
 
-        # Remove empty parameters
-        params = {k: v for k, v in params.items() if v != ""}
-
-        # If no engines are specified, set default engines
-        if 'engines' not in params:
-            params['engines'] = 'google'  # Default to 'google' or any preferred engine
-            logger.info("No engines specified. Defaulting to 'google'.")
-
-        # Headers for SearXNG request
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-            'Accept': 'application/json, text/javascript, */*; q=0.01',
-            'Accept-Language': 'en-US,en;q=0.5',
-            'Origin': 'https://shreyas094-searxng-local.hf.space',
-            'Referer': 'https://shreyas094-searxng-local.hf.space/',
-            'DNT': '1',
-            'Connection': 'keep-alive',
-            'Sec-Fetch-Dest': 'empty',
-            'Sec-Fetch-Mode': 'cors',
-            'Sec-Fetch-Site': 'same-origin',
-        }
+            # If no engines are specified, set default engines
+            if 'engines' not in params:
+                params['engines'] = 'google'  # Default to 'google' or any preferred engine
+                logger.info("No engines specified. Defaulting to 'google'.")
+    
+            # Headers for SearXNG request
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+                'Accept': 'application/json, text/javascript, */*; q=0.01',
+                'Accept-Language': 'en-US,en;q=0.5',
+                'Origin': 'https://shreyas094-searxng-local.hf.space',
+                'Referer': 'https://shreyas094-searxng-local.hf.space/',
+                'DNT': '1',
+                'Connection': 'keep-alive',
+                'Sec-Fetch-Dest': 'empty',
+                'Sec-Fetch-Mode': 'cors',
+                'Sec-Fetch-Site': 'same-origin',
+            }
 
         scraped_content = []
         page = 1
         while len(scraped_content) < num_results:
-            # Update params with current page
-            params['pageno'] = page
+            if not use_duckduckgo:
+                # Update params with current page
+                params['pageno'] = page
 
-            # Send request to SearXNG
-            logger.info(f"Sending request to SearXNG for query: {rephrased_query} (Page {page})")
-            session = requests_retry_session()
+                # Send request to SearXNG
+                logger.info(f"Sending request to SearXNG for query: {rephrased_query} (Page {page})")
+                session = requests_retry_session()
 
-            try:
-                if method.upper() == "GET":
-                    response = session.get(SEARXNG_URL, params=params, headers=headers, timeout=10, verify=certifi.where())
-                else:  # POST
-                    response = session.post(SEARXNG_URL, data=params, headers=headers, timeout=10, verify=certifi.where())
-                
-                response.raise_for_status()
-            except requests.exceptions.RequestException as e:
-                logger.error(f"Error during SearXNG request: {e}")
-                return f"An error occurred during the search request: {e}"
+                try:
+                    if method.upper() == "GET":
+                        response = session.get(SEARXNG_URL, params=params, headers=headers, timeout=10, verify=certifi.where())
+                    else:  # POST
+                        response = session.post(SEARXNG_URL, data=params, headers=headers, timeout=10, verify=certifi.where())
+                    
+                    response.raise_for_status()
+                except requests.exceptions.RequestException as e:
+                    logger.error(f"Error during SearXNG request: {e}")
+                    return f"An error occurred during the search request: {e}"
 
-            search_results = response.json()
-            logger.debug(f"SearXNG Response: {search_results}")
+                search_results = response.json()
+                logger.debug(f"SearXNG Response: {search_results}")
 
-            results = search_results.get('results', [])
-            if not results:
-                logger.warning(f"No more results returned from SearXNG on page {page}.")
-                break
+                results = search_results.get('results', [])
+                if not results:
+                    logger.warning(f"No more results returned from SearXNG on page {page}.")
+                    break
 
             for result in results:
                 if len(scraped_content) >= num_results:
                     break
         
-                url = result.get('url', '')
+                url = result.get('url', '') if not use_duckduckgo else result.get('href', '')
                 title = result.get('title', 'No title')
         
                 if not is_valid_url(url):
@@ -584,7 +586,10 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="bs4", max_cha
                 except Exception as e:
                     logger.error(f"Unexpected error while scraping {url}: {e}")
 
-            page += 1
+            if use_duckduckgo:
+                break  # DuckDuckGo search doesn't support pagination in this implementation
+            else:
+                page += 1
 
         if not scraped_content:
             logger.warning("No content scraped from search results.")