Update web_scraper.py

web_scraper.py

@@ -159,6 +159,7 @@ def scrape_site_content(query, num_sites=5):
                 try:
                     # Get the HTML content
                     logger.info(f"Trying {url} (attempt {attempt + 1}/{retries})")
+                    print(f"Trying {url} (attempt {attempt + 1}/{retries})")
                     response = requests.get(
                         url,
                         headers=headers,
@@ -186,6 +187,7 @@ def scrape_site_content(query, num_sites=5):
                     # Skip if not enough content
                     if len(text_content.split()) < 100:  # Skip if less than 100 words
                         logger.warning(f"Skipping {url} - not enough content")
+                        print(f"Skipping {url} - not HTML content")
                         break
 
                     # Extract all links (limit to first 10)