Update app.py

app.py

@@ -703,6 +703,36 @@ class ChatBot:
             logger.error(f'Error in search_and_summarize: {e}')
             return f"Error occurred: {str(e)}"
 
+     async def scrape_specific_urls(self, urls: List[str], max_chars: int) -> List[Dict[str, Any]]:
+        """Scrape specific URLs provided by the user."""
+        logger.info(f'Scraping specific URLs: {urls}')
+        try:
+            # Create dummy results structure expected by scrape_urls_parallel
+            results = [{"url": url} for url in urls]
+            valid_results = await scrape_urls_parallel(results, max_chars)
+            
+            if not valid_results:
+                logger.info("No valid content found from provided URLs")
+                return []
+            
+            processed_articles = []
+            for result, article in valid_results:
+                if article:
+                    processed_articles.append({
+                        "url": result["url"],
+                        "title": urlparse(result["url"]).netloc,  # Use domain as title if not available
+                        "content": article["content"],
+                        "publish_date": article["publish_date"],
+                        "score": 1.0,  # Direct URL scraping, so score is 1.0
+                        "engine": "direct_url"
+                    })
+            
+            return processed_articles
+            
+        except Exception as e:
+            logger.error(f'Error scraping specific URLs: {e}')
+            return []
+    
     async def get_response(self,
                           query: str,
                           history: List[List[str]],