app.py

import gradio as gr
import requests
from bs4 import BeautifulSoup
from openai import OpenAI
import json
import re
from urllib.parse import urljoin, urlparse
import time
import urllib3
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
import ssl

# Disable SSL warnings
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

class WebScrapingTool:
    def __init__(self):
        self.client = None
        self.system_prompt = """You are a specialized web data extraction assistant. Your core purpose is to browse and analyze the content of web pages based on user instructions, and return structured or unstructured information from the provided URL. Your capabilities include:
1. Navigating and reading web page content from a given URL.
2. Extracting textual content including headings, paragraphs, lists, and metadata.
3. Identifying and extracting HTML tables and presenting them in a clean, structured format.
4. Creating new, custom tables based on user queries by processing, reorganizing, or filtering the content found on the source page.
You must always follow these guidelines:
- Accurately extract and summarize both structured (tables, lists) and unstructured (paragraphs, articles) content.
- Clearly separate different types of data (e.g., summaries, tables, bullet points).
- When extracting textual content:
  - Maintain original meaning, structure, and tone.
  - Capture all relevant sections based on user instructions (e.g., only the "Overview" or "Methodology" sections).
- When extracting tables:
  - Preserve headers and align row data correctly.
  - Identify and differentiate multiple tables, if present.
- When creating custom tables:
  - Include only the relevant columns as per the user request.
  - Sort, filter, and reorganize data accordingly.
  - Use clear and consistent headers.
You must not hallucinate or infer data not present on the page. If content is missing, unclear, or restricted, say so explicitly.
Always respond based on the actual content from the provided link. If the page fails to load or cannot be accessed, inform the user immediately.
Your role is to act as an intelligent browser and data interpreter — able to read and reshape any web content to meet user needs."""

    def setup_client(self, api_key):
        """Initialize OpenAI client with OpenRouter"""
        try:
            self.client = OpenAI(
                base_url="https://openrouter.ai/api/v1",
                api_key=api_key,
            )
            return True, "API client initialized successfully!"
        except Exception as e:
            return False, f"Failed to initialize API client: {str(e)}"

    def create_session(self):
        """Create a robust session with retry strategy and proper headers"""
        session = requests.Session()
        
        # Define retry strategy with fixed parameter name
        retry_strategy = Retry(
            total=3,
            status_forcelist=[429, 500, 502, 503, 504],
            allowed_methods=["HEAD", "GET", "OPTIONS"],  # Fixed: changed from method_whitelist
            backoff_factor=1
        )
        
        # Mount adapter with retry strategy
        adapter = HTTPAdapter(max_retries=retry_strategy)
        session.mount("http://", adapter)
        session.mount("https://", adapter)
        
        # Set comprehensive headers to mimic real browser
        session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Accept-Language': 'en-US,en;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'DNT': '1',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Sec-Fetch-User': '?1',
            'Cache-Control': 'max-age=0'
        })
        
        return session

    def scrape_webpage(self, url):
        """Scrape webpage content with enhanced error handling and timeouts"""
        try:
            session = self.create_session()
            
            # Multiple timeout attempts with increasing duration
            timeout_attempts = [15, 30, 45]
            response = None
            
            for timeout in timeout_attempts:
                try:
                    print(f"Attempting to fetch {url} with {timeout}s timeout...")
                    
                    response = session.get(
                        url, 
                        timeout=timeout,
                        verify=False,  # Disable SSL verification for problematic sites
                        allow_redirects=True,
                        stream=False
                    )
                    
                    response.raise_for_status()
                    break
                    
                except requests.exceptions.Timeout:
                    if timeout == timeout_attempts[-1]:  # Last attempt
                        return {
                            'success': False,
                            'error': f"Connection timed out after multiple attempts. The website may be slow or blocking automated requests."
                        }
                    continue
                except requests.exceptions.SSLError:
                    # Try with different SSL context
                    try:
                        response = session.get(
                            url, 
                            timeout=timeout,
                            verify=False,
                            allow_redirects=True
                        )
                        response.raise_for_status()
                        break
                    except:
                        continue
                except requests.exceptions.RequestException as e:
                    if timeout == timeout_attempts[-1]:  # Last attempt
                        return {
                            'success': False,
                            'error': f"Request failed: {str(e)}"
                        }
                    continue
            
            # Check if we got a response
            if response is None:
                return {
                    'success': False,
                    'error': "Failed to establish connection after multiple attempts"
                }
            
            # Check content type
            content_type = response.headers.get('content-type', '').lower()
            if 'text/html' not in content_type and 'text/plain' not in content_type:
                return {
                    'success': False,
                    'error': f"Invalid content type: {content_type}. Expected HTML content."
                }
            
            # Parse HTML content
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # Remove unwanted elements
            for element in soup(["script", "style", "nav", "footer", "header", "aside", "noscript", "iframe"]):
                element.decompose()
            
            # Remove elements with common ad/tracking classes
            ad_classes = ['ad', 'advertisement', 'banner', 'popup', 'modal', 'cookie', 'newsletter']
            for class_name in ad_classes:
                for element in soup.find_all(class_=re.compile(class_name, re.I)):
                    element.decompose()
            
            # Extract text content
            text_content = soup.get_text(separator=' ', strip=True)
            
            # Clean up text - remove extra whitespace
            text_content = re.sub(r'\s+', ' ', text_content)
            text_content = text_content.strip()
            
            # Extract tables with improved structure
            tables = []
            for i, table in enumerate(soup.find_all('table')):
                table_data = []
                headers = []
                
                # Try to find headers in various ways
                header_row = table.find('thead')
                if header_row:
                    header_row = header_row.find('tr')
                else:
                    header_row = table.find('tr')
                
                if header_row:
                    headers = []
                    for th in header_row.find_all(['th', 'td']):
                        header_text = th.get_text(strip=True)
                        headers.append(header_text if header_text else f"Column_{len(headers)+1}")
                
                # Extract all rows (skip header if it was already processed)
                rows = table.find_all('tr')
                start_idx = 1 if header_row and header_row in rows else 0
                
                for row in rows[start_idx:]:
                    cells = row.find_all(['td', 'th'])
                    if cells:
                        row_data = []
                        for cell in cells:
                            cell_text = cell.get_text(strip=True)
                            row_data.append(cell_text)
                        
                        if row_data and any(cell.strip() for cell in row_data):  # Skip empty rows
                            table_data.append(row_data)
                
                if table_data:
                    # Ensure headers match data columns
                    max_cols = max(len(row) for row in table_data) if table_data else 0
                    if len(headers) < max_cols:
                        headers.extend([f"Column_{i+1}" for i in range(len(headers), max_cols)])
                    elif len(headers) > max_cols:
                        headers = headers[:max_cols]
                    
                    tables.append({
                        'id': i + 1,
                        'headers': headers,
                        'data': table_data[:50]  # Limit rows to prevent overwhelming
                    })
            
            # Extract metadata
            title = soup.title.string.strip() if soup.title and soup.title.string else "No title found"
            
            # Extract meta description
            meta_desc = ""
            desc_tag = soup.find('meta', attrs={'name': 'description'})
            if desc_tag and desc_tag.get('content'):
                meta_desc = desc_tag['content'].strip()
            
            return {
                'success': True,
                'text': text_content[:20000],  # Limit text length
                'tables': tables,
                'title': title,
                'meta_description': meta_desc,
                'url': url,
                'content_length': len(text_content)
            }
            
        except requests.exceptions.ConnectionError as e:
            return {
                'success': False,
                'error': f"Connection failed: {str(e)}. The website may be down or blocking requests."
            }
        except requests.exceptions.HTTPError as e:
            return {
                'success': False,
                'error': f"HTTP Error {e.response.status_code}: {e.response.reason}"
            }
        except requests.exceptions.RequestException as e:
            return {
                'success': False,
                'error': f"Request failed: {str(e)}"
            }
        except Exception as e:
            return {
                'success': False,
                'error': f"Unexpected error while processing webpage: {str(e)}"
            }

    def analyze_content(self, scraped_data, user_query, api_key):
        """Analyze scraped content using DeepSeek V3"""
        if not self.client:
            success, message = self.setup_client(api_key)
            if not success:
                return f"Error: {message}"
        
        if not scraped_data['success']:
            return f"Error scraping webpage: {scraped_data['error']}"
        
        # Prepare content for AI analysis
        content_text = f"""
WEBPAGE ANALYSIS REQUEST
========================
URL: {scraped_data['url']}
Title: {scraped_data['title']}
Content Length: {scraped_data['content_length']} characters
Tables Found: {len(scraped_data['tables'])}
META DESCRIPTION:
{scraped_data['meta_description']}
MAIN CONTENT:
{scraped_data['text']}
"""
        
        if scraped_data['tables']:
            content_text += f"\n\nSTRUCTURED DATA - {len(scraped_data['tables'])} TABLE(S) FOUND:\n"
            content_text += "=" * 50 + "\n"
            
            for table in scraped_data['tables']:
                content_text += f"\nTABLE {table['id']}:\n"
                content_text += f"Headers: {' | '.join(table['headers'])}\n"
                content_text += "-" * 50 + "\n"
                
                for i, row in enumerate(table['data'][:10]):  # Show first 10 rows
                    content_text += f"Row {i+1}: {' | '.join(str(cell) for cell in row)}\n"
                
                if len(table['data']) > 10:
                    content_text += f"... and {len(table['data']) - 10} more rows\n"
                content_text += "\n"
        
        try:
            completion = self.client.chat.completions.create(
                extra_headers={
                    "HTTP-Referer": "https://gradio-web-scraper.com",
                    "X-Title": "AI Web Scraping Tool",
                },
                model="deepseek/deepseek-chat-v3-0324:free",
                messages=[
                    {"role": "system", "content": self.system_prompt},
                    {"role": "user", "content": f"{content_text}\n\nUSER REQUEST:\n{user_query}\n\nPlease analyze the above webpage content and fulfill the user's request. Be thorough and accurate."}
                ],
                temperature=0.1,
                max_tokens=4000
            )
            
            return completion.choices[0].message.content
            
        except Exception as e:
            return f"Error analyzing content with AI: {str(e)}"

def create_interface():
    tool = WebScrapingTool()
    
    def process_request(api_key, url, user_query):
        if not api_key.strip():
            return "❌ Please enter your OpenRouter API key"
        
        if not url.strip():
            return "❌ Please enter a valid URL"
        
        if not user_query.strip():
            return "❌ Please enter your analysis query"
        
        # Validate URL format
        if not url.startswith(('http://', 'https://')):
            url = 'https://' + url
        
        # Add progress updates
        yield "🔄 Initializing web scraper..."
        time.sleep(0.5)
        
        yield "🌐 Fetching webpage content (this may take a moment)..."
        
        # Scrape webpage
        scraped_data = tool.scrape_webpage(url)
        
        if not scraped_data['success']:
            yield f"❌ Scraping Failed: {scraped_data['error']}"
            return
        
        yield f"✅ Successfully scraped webpage!\n📄 Title: {scraped_data['title']}\n📊 Found {len(scraped_data['tables'])} tables\n📝 Content: {scraped_data['content_length']} characters\n\n🤖 Analyzing content with DeepSeek V3..."
        
        # Analyze content
        result = tool.analyze_content(scraped_data, user_query, api_key)
        
        yield f"✅ Analysis Complete!\n{'='*50}\n\n{result}"
    
    # Create Gradio interface
    with gr.Blocks(title="AI Web Scraping Tool", theme=gr.themes.Soft()) as app:
        gr.Markdown("""
        # 🤖 AI Web Scraping Tool
        ### Powered by DeepSeek V3 & OpenRouter
        
        Extract and analyze web content using advanced AI. The tool handles timeouts, SSL issues, and provides robust scraping capabilities.
        """)
        
        with gr.Row():
            with gr.Column(scale=2):
                api_key_input = gr.Textbox(
                    label="🔑 OpenRouter API Key",
                    placeholder="Enter your OpenRouter API key here...",
                    type="password",
                    info="Get your free API key from openrouter.ai"
                )
                
                url_input = gr.Textbox(
                    label="🌐 Website URL",
                    placeholder="https://example.com or just example.com",
                    info="Enter the URL you want to scrape and analyze"
                )
                
                query_input = gr.Textbox(
                    label="📝 Analysis Query",
                    placeholder="What do you want to extract? (e.g., 'Extract main points and create a summary table')",
                    lines=4,
                    info="Describe what information you want to extract from the webpage"
                )
                
                with gr.Row():
                    analyze_btn = gr.Button("🚀 Analyze Website", variant="primary", size="lg")
                    clear_btn = gr.Button("🗑️ Clear All", variant="secondary")
            
            with gr.Column(scale=3):
                output = gr.Textbox(
                    label="📊 Analysis Results",
                    lines=25,
                    max_lines=40,
                    show_copy_button=True,
                    interactive=False,
                    placeholder="Results will appear here after analysis..."
                )
        
        # Tips and Examples
        with gr.Accordion("💡 Usage Tips & Examples", open=False):
            gr.Markdown("""
            ### 🎯 Example Analysis Queries:
            - **Data Extraction**: *"Extract all numerical data and organize it in a table format"*
            - **Content Summary**: *"Summarize the main points in bullet format with key statistics"*
            - **Table Processing**: *"Find all tables and convert them to a single consolidated format"*
            - **Specific Information**: *"Extract contact information, prices, or product details"*
            - **Comparison**: *"Compare different items/options mentioned and create a comparison table"*
            
            ### 🔧 Technical Notes:
            - **Multiple Timeouts**: Tool tries 15s, 30s, then 45s timeouts automatically
            - **SSL Handling**: Bypasses SSL issues for problematic websites
            - **Content Filtering**: Removes ads, popups, and unnecessary elements
            - **Table Detection**: Automatically finds and structures tabular data
            - **Error Recovery**: Handles connection issues and provides clear error messages
            
            ### 🌐 Works Well With:
            - News websites (BBC, CNN, Reuters)
            - Government sites (IMF, WHO, official statistics)
            - Wikipedia and educational content
            - E-commerce product pages
            - Financial data sites (Yahoo Finance, MarketWatch)
            - Research papers and academic sites
            
            ## 🧪 **Test Scenarios**
            
            ### **1. News & Media Sites**
            ```
            URL: https://www.bbc.com/news
            Query: Extract the top 5 news headlines with their summaries and create a table with columns: Headline, Category, Summary
            ```
            
            ```
            URL: https://edition.cnn.com
            Query: Find all breaking news items and organize them by topic/region in a structured format
            ```
            
            ### **2. Financial Data Sites**
            ```
            URL: https://finance.yahoo.com/quote/AAPL
            Query: Extract Apple stock information including current price, daily change, market cap, and any financial metrics into a summary table
            ```
            
            ```
            URL: https://www.marketwatch.com/investing/stock/tsla
            Query: Create a table with Tesla's key financial metrics: price, change, volume, market cap, P/E ratio
            ```
            
            ### **3. E-commerce & Product Pages**
            ```
            URL: https://www.amazon.com/dp/B08N5WRWNW
            Query: Extract product details including name, price, ratings, key features, and specifications in a structured format
            ```
            
            ```
            URL: https://www.ebay.com/itm/123456789
            Query: Extract item details, price, seller information, and shipping details into a comparison-ready table
            ```
            
            ### **4. Educational & Reference Sites**
            ```
            URL: https://en.wikipedia.org/wiki/Artificial_intelligence
            Query: Extract the main definition, history timeline, and applications of AI. Create separate sections for each topic.
            ```
            
            ```
            URL: https://en.wikipedia.org/wiki/List_of_countries_by_population
            Query: Extract the population data table and create a new table showing top 10 most populous countries with their population and growth rate
            ```
            
            ### **5. Government & Official Statistics**
            ```
            URL: https://www.who.int/emergencies/diseases/novel-coronavirus-2019/situation-reports
            Query: Extract the latest COVID-19 statistics and create a summary table with key global figures
            ```
            
            ```
            URL: https://www.census.gov/quickfacts
            Query: Extract key demographic statistics for the United States and organize them into categories: Population, Economy, Geography
            ```
            
            ### **6. Technology & Business News**
            ```
            URL: https://techcrunch.com
            Query: Find the latest startup funding news and create a table with: Company Name, Funding Amount, Investors, Industry
            ```
            
            ```
            URL: https://www.reuters.com/technology
            Query: Extract top technology news and summarize each story in 2-3 sentences with key points
            ```
            
            ### **7. Scientific & Research Sites**
            ```
            URL: https://www.nature.com/articles
            Query: Extract recent scientific article titles, authors, and abstracts. Create a summary table organized by research field
            ```
            
            ```
            URL: https://pubmed.ncbi.nlm.nih.gov/trending
            Query: Find trending medical research topics and create a list with brief descriptions of each study's findings
            ```
            
            ### **8. Sports & Entertainment**
            ```
            URL: https://www.espn.com/nba/standings
            Query: Extract NBA team standings and create a table with: Team, Wins, Losses, Win Percentage, Conference Position
            ```
            
            ```
            URL: https://www.imdb.com/chart/top
            Query: Extract the top 10 movies from IMDb's top 250 list with ratings, year, and brief description
            ```
            
            ### **9. Weather & Environmental Data**
            ```
            URL: https://weather.com/weather/today
            Query: Extract current weather conditions and forecast data. Create a summary with temperature, conditions, and weekly outlook
            ```
            
            ### **10. Real Estate & Property**
            ```
            URL: https://www.zillow.com/homes/for_sale
            Query: Extract property listings with prices, locations, square footage, and key features into a comparison table
            ```
            
            ## 🎯 **Quick Test Samples (Copy & Paste Ready)**
            
            ### **Simple Test:**
            ```
            URL: https://httpbin.org/html
            Query: Extract all text content and identify the page structure
            ```
            
            ### **Table Extraction Test:**
            ```
            URL: https://www.w3schools.com/html/html_tables.asp
            Query: Find all HTML tables on this page and convert them to a structured format with proper headers
            ```
            
            ### **Complex Analysis Test:**
            ```
            URL: https://www.sec.gov/edgar/browse/?CIK=320193
            Query: Extract Apple Inc.'s recent SEC filings and create a table with: Filing Date, Document Type, Description
            ```
            
            ### **International Site Test:**
            ```
            URL: https://www.bbc.co.uk/weather
            Query: Extract UK weather information and create a regional breakdown of current conditions
            ```
            
            ## 🔍 **Testing Tips:**
            
            1. **Start Simple**: Begin with basic sites like Wikipedia or news sites
            2. **Test Error Handling**: Try invalid URLs to see error messages
            3. **Check Timeouts**: Use slow-loading sites to test timeout handling
            4. **Verify Tables**: Test sites with different table structures
            5. **Content Variety**: Try different content types (news, data, products)
            
            ## 🚨 **Sites That May Have Issues:**
            - Social media sites (require login)
            - Sites with heavy JavaScript (may have limited content)
            - Sites with aggressive bot protection
            - Password-protected pages
            
            ## ✅ **Reliable Test Sites:**
            - Wikipedia (excellent for tables and structured content)
            - BBC News (good for text extraction)
            - Government sites (.gov domains)
            - W3Schools (great for HTML table testing)
            - HttpBin (perfect for testing basic functionality)
            
            Start with the simpler tests and gradually move to more complex scenarios to fully evaluate your tool's capabilities!
            """)
        
        # Event handlers
        analyze_btn.click(
            fn=process_request,
            inputs=[api_key_input, url_input, query_input],
            outputs=output,
            show_progress=True
        )
        
        clear_btn.click(
            fn=lambda: ("", "", "", ""),
            outputs=[api_key_input, url_input, query_input, output]
        )
    
    return app

if __name__ == "__main__":
    # Create and launch the app
    app = create_interface()
    
    # Launch with enhanced configuration
    app.launch(
        share=True
    )