Create v1.txt

v1.txt

@@ -0,0 +1,604 @@
+import gradio as gr
+import requests
+from bs4 import BeautifulSoup
+from openai import OpenAI
+import json
+import re
+from urllib.parse import urljoin, urlparse
+import time
+import urllib3
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+import ssl
+
+# Disable SSL warnings
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+class WebScrapingTool:
+    def __init__(self):
+        self.client = None
+        self.system_prompt = """You are a specialized web data extraction assistant. Your core purpose is to browse and analyze the content of web pages based on user instructions, and return structured or unstructured information from the provided URL. Your capabilities include:
+1. Navigating and reading web page content from a given URL.
+2. Extracting textual content including headings, paragraphs, lists, and metadata.
+3. Identifying and extracting HTML tables and presenting them in a clean, structured format.
+4. Creating new, custom tables based on user queries by processing, reorganizing, or filtering the content found on the source page.
+You must always follow these guidelines:
+- Accurately extract and summarize both structured (tables, lists) and unstructured (paragraphs, articles) content.
+- Clearly separate different types of data (e.g., summaries, tables, bullet points).
+- When extracting textual content:
+  - Maintain original meaning, structure, and tone.
+  - Capture all relevant sections based on user instructions (e.g., only the "Overview" or "Methodology" sections).
+- When extracting tables:
+  - Preserve headers and align row data correctly.
+  - Identify and differentiate multiple tables, if present.
+- When creating custom tables:
+  - Include only the relevant columns as per the user request.
+  - Sort, filter, and reorganize data accordingly.
+  - Use clear and consistent headers.
+You must not hallucinate or infer data not present on the page. If content is missing, unclear, or restricted, say so explicitly.
+Always respond based on the actual content from the provided link. If the page fails to load or cannot be accessed, inform the user immediately.
+Your role is to act as an intelligent browser and data interpreter — able to read and reshape any web content to meet user needs."""
+
+    def setup_client(self, api_key):
+        """Initialize OpenAI client with OpenRouter"""
+        try:
+            self.client = OpenAI(
+                base_url="https://openrouter.ai/api/v1",
+                api_key=api_key,
+            )
+            return True, "API client initialized successfully!"
+        except Exception as e:
+            return False, f"Failed to initialize API client: {str(e)}"
+
+    def create_session(self):
+        """Create a robust session with retry strategy and proper headers"""
+        session = requests.Session()
+        
+        # Define retry strategy with fixed parameter name
+        retry_strategy = Retry(
+            total=3,
+            status_forcelist=[429, 500, 502, 503, 504],
+            allowed_methods=["HEAD", "GET", "OPTIONS"],  # Fixed: changed from method_whitelist
+            backoff_factor=1
+        )
+        
+        # Mount adapter with retry strategy
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        session.mount("http://", adapter)
+        session.mount("https://", adapter)
+        
+        # Set comprehensive headers to mimic real browser
+        session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+            'Accept-Language': 'en-US,en;q=0.9',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'DNT': '1',
+            'Connection': 'keep-alive',
+            'Upgrade-Insecure-Requests': '1',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'none',
+            'Sec-Fetch-User': '?1',
+            'Cache-Control': 'max-age=0'
+        })
+        
+        return session
+
+    def scrape_webpage(self, url):
+        """Scrape webpage content with enhanced error handling and timeouts"""
+        try:
+            session = self.create_session()
+            
+            # Multiple timeout attempts with increasing duration
+            timeout_attempts = [15, 30, 45]
+            response = None
+            
+            for timeout in timeout_attempts:
+                try:
+                    print(f"Attempting to fetch {url} with {timeout}s timeout...")
+                    
+                    response = session.get(
+                        url, 
+                        timeout=timeout,
+                        verify=False,  # Disable SSL verification for problematic sites
+                        allow_redirects=True,
+                        stream=False
+                    )
+                    
+                    response.raise_for_status()
+                    break
+                    
+                except requests.exceptions.Timeout:
+                    if timeout == timeout_attempts[-1]:  # Last attempt
+                        return {
+                            'success': False,
+                            'error': f"Connection timed out after multiple attempts. The website may be slow or blocking automated requests."
+                        }
+                    continue
+                except requests.exceptions.SSLError:
+                    # Try with different SSL context
+                    try:
+                        response = session.get(
+                            url, 
+                            timeout=timeout,
+                            verify=False,
+                            allow_redirects=True
+                        )
+                        response.raise_for_status()
+                        break
+                    except:
+                        continue
+                except requests.exceptions.RequestException as e:
+                    if timeout == timeout_attempts[-1]:  # Last attempt
+                        return {
+                            'success': False,
+                            'error': f"Request failed: {str(e)}"
+                        }
+                    continue
+            
+            # Check if we got a response
+            if response is None:
+                return {
+                    'success': False,
+                    'error': "Failed to establish connection after multiple attempts"
+                }
+            
+            # Check content type
+            content_type = response.headers.get('content-type', '').lower()
+            if 'text/html' not in content_type and 'text/plain' not in content_type:
+                return {
+                    'success': False,
+                    'error': f"Invalid content type: {content_type}. Expected HTML content."
+                }
+            
+            # Parse HTML content
+            soup = BeautifulSoup(response.content, 'html.parser')
+            
+            # Remove unwanted elements
+            for element in soup(["script", "style", "nav", "footer", "header", "aside", "noscript", "iframe"]):
+                element.decompose()
+            
+            # Remove elements with common ad/tracking classes
+            ad_classes = ['ad', 'advertisement', 'banner', 'popup', 'modal', 'cookie', 'newsletter']
+            for class_name in ad_classes:
+                for element in soup.find_all(class_=re.compile(class_name, re.I)):
+                    element.decompose()
+            
+            # Extract text content
+            text_content = soup.get_text(separator=' ', strip=True)
+            
+            # Clean up text - remove extra whitespace
+            text_content = re.sub(r'\s+', ' ', text_content)
+            text_content = text_content.strip()
+            
+            # Extract tables with improved structure
+            tables = []
+            for i, table in enumerate(soup.find_all('table')):
+                table_data = []
+                headers = []
+                
+                # Try to find headers in various ways
+                header_row = table.find('thead')
+                if header_row:
+                    header_row = header_row.find('tr')
+                else:
+                    header_row = table.find('tr')
+                
+                if header_row:
+                    headers = []
+                    for th in header_row.find_all(['th', 'td']):
+                        header_text = th.get_text(strip=True)
+                        headers.append(header_text if header_text else f"Column_{len(headers)+1}")
+                
+                # Extract all rows (skip header if it was already processed)
+                rows = table.find_all('tr')
+                start_idx = 1 if header_row and header_row in rows else 0
+                
+                for row in rows[start_idx:]:
+                    cells = row.find_all(['td', 'th'])
+                    if cells:
+                        row_data = []
+                        for cell in cells:
+                            cell_text = cell.get_text(strip=True)
+                            row_data.append(cell_text)
+                        
+                        if row_data and any(cell.strip() for cell in row_data):  # Skip empty rows
+                            table_data.append(row_data)
+                
+                if table_data:
+                    # Ensure headers match data columns
+                    max_cols = max(len(row) for row in table_data) if table_data else 0
+                    if len(headers) < max_cols:
+                        headers.extend([f"Column_{i+1}" for i in range(len(headers), max_cols)])
+                    elif len(headers) > max_cols:
+                        headers = headers[:max_cols]
+                    
+                    tables.append({
+                        'id': i + 1,
+                        'headers': headers,
+                        'data': table_data[:50]  # Limit rows to prevent overwhelming
+                    })
+            
+            # Extract metadata
+            title = soup.title.string.strip() if soup.title and soup.title.string else "No title found"
+            
+            # Extract meta description
+            meta_desc = ""
+            desc_tag = soup.find('meta', attrs={'name': 'description'})
+            if desc_tag and desc_tag.get('content'):
+                meta_desc = desc_tag['content'].strip()
+            
+            return {
+                'success': True,
+                'text': text_content[:20000],  # Limit text length
+                'tables': tables,
+                'title': title,
+                'meta_description': meta_desc,
+                'url': url,
+                'content_length': len(text_content)
+            }
+            
+        except requests.exceptions.ConnectionError as e:
+            return {
+                'success': False,
+                'error': f"Connection failed: {str(e)}. The website may be down or blocking requests."
+            }
+        except requests.exceptions.HTTPError as e:
+            return {
+                'success': False,
+                'error': f"HTTP Error {e.response.status_code}: {e.response.reason}"
+            }
+        except requests.exceptions.RequestException as e:
+            return {
+                'success': False,
+                'error': f"Request failed: {str(e)}"
+            }
+        except Exception as e:
+            return {
+                'success': False,
+                'error': f"Unexpected error while processing webpage: {str(e)}"
+            }
+
+    def analyze_content(self, scraped_data, user_query, api_key):
+        """Analyze scraped content using DeepSeek V3"""
+        if not self.client:
+            success, message = self.setup_client(api_key)
+            if not success:
+                return f"Error: {message}"
+        
+        if not scraped_data['success']:
+            return f"Error scraping webpage: {scraped_data['error']}"
+        
+        # Prepare content for AI analysis
+        content_text = f"""
+WEBPAGE ANALYSIS REQUEST
+========================
+URL: {scraped_data['url']}
+Title: {scraped_data['title']}
+Content Length: {scraped_data['content_length']} characters
+Tables Found: {len(scraped_data['tables'])}
+META DESCRIPTION:
+{scraped_data['meta_description']}
+MAIN CONTENT:
+{scraped_data['text']}
+"""
+        
+        if scraped_data['tables']:
+            content_text += f"\n\nSTRUCTURED DATA - {len(scraped_data['tables'])} TABLE(S) FOUND:\n"
+            content_text += "=" * 50 + "\n"
+            
+            for table in scraped_data['tables']:
+                content_text += f"\nTABLE {table['id']}:\n"
+                content_text += f"Headers: {' | '.join(table['headers'])}\n"
+                content_text += "-" * 50 + "\n"
+                
+                for i, row in enumerate(table['data'][:10]):  # Show first 10 rows
+                    content_text += f"Row {i+1}: {' | '.join(str(cell) for cell in row)}\n"
+                
+                if len(table['data']) > 10:
+                    content_text += f"... and {len(table['data']) - 10} more rows\n"
+                content_text += "\n"
+        
+        try:
+            completion = self.client.chat.completions.create(
+                extra_headers={
+                    "HTTP-Referer": "https://gradio-web-scraper.com",
+                    "X-Title": "AI Web Scraping Tool",
+                },
+                model="deepseek/deepseek-chat-v3-0324:free",
+                messages=[
+                    {"role": "system", "content": self.system_prompt},
+                    {"role": "user", "content": f"{content_text}\n\nUSER REQUEST:\n{user_query}\n\nPlease analyze the above webpage content and fulfill the user's request. Be thorough and accurate."}
+                ],
+                temperature=0.1,
+                max_tokens=4000
+            )
+            
+            return completion.choices[0].message.content
+            
+        except Exception as e:
+            return f"Error analyzing content with AI: {str(e)}"
+
+def create_interface():
+    tool = WebScrapingTool()
+    
+    def process_request(api_key, url, user_query):
+        if not api_key.strip():
+            return "❌ Please enter your OpenRouter API key"
+        
+        if not url.strip():
+            return "❌ Please enter a valid URL"
+        
+        if not user_query.strip():
+            return "❌ Please enter your analysis query"
+        
+        # Validate URL format
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url
+        
+        # Add progress updates
+        yield "🔄 Initializing web scraper..."
+        time.sleep(0.5)
+        
+        yield "🌐 Fetching webpage content (this may take a moment)..."
+        
+        # Scrape webpage
+        scraped_data = tool.scrape_webpage(url)
+        
+        if not scraped_data['success']:
+            yield f"❌ Scraping Failed: {scraped_data['error']}"
+            return
+        
+        yield f"✅ Successfully scraped webpage!\n📄 Title: {scraped_data['title']}\n📊 Found {len(scraped_data['tables'])} tables\n📝 Content: {scraped_data['content_length']} characters\n\n🤖 Analyzing content with DeepSeek V3..."
+        
+        # Analyze content
+        result = tool.analyze_content(scraped_data, user_query, api_key)
+        
+        yield f"✅ Analysis Complete!\n{'='*50}\n\n{result}"
+    
+    # Create Gradio interface
+    with gr.Blocks(title="AI Web Scraping Tool", theme=gr.themes.Soft()) as app:
+        gr.Markdown("""
+        # 🤖 AI Web Scraping Tool
+        ### Powered by DeepSeek V3 & OpenRouter
+        
+        Extract and analyze web content using advanced AI. The tool handles timeouts, SSL issues, and provides robust scraping capabilities.
+        """)
+        
+        with gr.Row():
+            with gr.Column(scale=2):
+                api_key_input = gr.Textbox(
+                    label="🔑 OpenRouter API Key",
+                    placeholder="Enter your OpenRouter API key here...",
+                    type="password",
+                    info="Get your free API key from openrouter.ai"
+                )
+                
+                url_input = gr.Textbox(
+                    label="🌐 Website URL",
+                    placeholder="https://example.com or just example.com",
+                    info="Enter the URL you want to scrape and analyze"
+                )
+                
+                query_input = gr.Textbox(
+                    label="📝 Analysis Query",
+                    placeholder="What do you want to extract? (e.g., 'Extract main points and create a summary table')",
+                    lines=4,
+                    info="Describe what information you want to extract from the webpage"
+                )
+                
+                with gr.Row():
+                    analyze_btn = gr.Button("🚀 Analyze Website", variant="primary", size="lg")
+                    clear_btn = gr.Button("🗑️ Clear All", variant="secondary")
+            
+            with gr.Column(scale=3):
+                output = gr.Textbox(
+                    label="📊 Analysis Results",
+                    lines=25,
+                    max_lines=40,
+                    show_copy_button=True,
+                    interactive=False,
+                    placeholder="Results will appear here after analysis..."
+                )
+        
+        # Tips and Examples
+        with gr.Accordion("💡 Usage Tips & Examples", open=False):
+            gr.Markdown("""
+            ### 🎯 Example Analysis Queries:
+            - **Data Extraction**: *"Extract all numerical data and organize it in a table format"*
+            - **Content Summary**: *"Summarize the main points in bullet format with key statistics"*
+            - **Table Processing**: *"Find all tables and convert them to a single consolidated format"*
+            - **Specific Information**: *"Extract contact information, prices, or product details"*
+            - **Comparison**: *"Compare different items/options mentioned and create a comparison table"*
+            
+            ### 🔧 Technical Notes:
+            - **Multiple Timeouts**: Tool tries 15s, 30s, then 45s timeouts automatically
+            - **SSL Handling**: Bypasses SSL issues for problematic websites
+            - **Content Filtering**: Removes ads, popups, and unnecessary elements
+            - **Table Detection**: Automatically finds and structures tabular data
+            - **Error Recovery**: Handles connection issues and provides clear error messages
+            
+            ### 🌐 Works Well With:
+            - News websites (BBC, CNN, Reuters)
+            - Government sites (IMF, WHO, official statistics)
+            - Wikipedia and educational content
+            - E-commerce product pages
+            - Financial data sites (Yahoo Finance, MarketWatch)
+            - Research papers and academic sites
+            
+            ## 🧪 **Test Scenarios**
+            
+            ### **1. News & Media Sites**
+            ```
+            URL: https://www.bbc.com/news
+            Query: Extract the top 5 news headlines with their summaries and create a table with columns: Headline, Category, Summary
+            ```
+            
+            ```
+            URL: https://edition.cnn.com
+            Query: Find all breaking news items and organize them by topic/region in a structured format
+            ```
+            
+            ### **2. Financial Data Sites**
+            ```
+            URL: https://finance.yahoo.com/quote/AAPL
+            Query: Extract Apple stock information including current price, daily change, market cap, and any financial metrics into a summary table
+            ```
+            
+            ```
+            URL: https://www.marketwatch.com/investing/stock/tsla
+            Query: Create a table with Tesla's key financial metrics: price, change, volume, market cap, P/E ratio
+            ```
+            
+            ### **3. E-commerce & Product Pages**
+            ```
+            URL: https://www.amazon.com/dp/B08N5WRWNW
+            Query: Extract product details including name, price, ratings, key features, and specifications in a structured format
+            ```
+            
+            ```
+            URL: https://www.ebay.com/itm/123456789
+            Query: Extract item details, price, seller information, and shipping details into a comparison-ready table
+            ```
+            
+            ### **4. Educational & Reference Sites**
+            ```
+            URL: https://en.wikipedia.org/wiki/Artificial_intelligence
+            Query: Extract the main definition, history timeline, and applications of AI. Create separate sections for each topic.
+            ```
+            
+            ```
+            URL: https://en.wikipedia.org/wiki/List_of_countries_by_population
+            Query: Extract the population data table and create a new table showing top 10 most populous countries with their population and growth rate
+            ```
+            
+            ### **5. Government & Official Statistics**
+            ```
+            URL: https://www.who.int/emergencies/diseases/novel-coronavirus-2019/situation-reports
+            Query: Extract the latest COVID-19 statistics and create a summary table with key global figures
+            ```
+            
+            ```
+            URL: https://www.census.gov/quickfacts
+            Query: Extract key demographic statistics for the United States and organize them into categories: Population, Economy, Geography
+            ```
+            
+            ### **6. Technology & Business News**
+            ```
+            URL: https://techcrunch.com
+            Query: Find the latest startup funding news and create a table with: Company Name, Funding Amount, Investors, Industry
+            ```
+            
+            ```
+            URL: https://www.reuters.com/technology
+            Query: Extract top technology news and summarize each story in 2-3 sentences with key points
+            ```
+            
+            ### **7. Scientific & Research Sites**
+            ```
+            URL: https://www.nature.com/articles
+            Query: Extract recent scientific article titles, authors, and abstracts. Create a summary table organized by research field
+            ```
+            
+            ```
+            URL: https://pubmed.ncbi.nlm.nih.gov/trending
+            Query: Find trending medical research topics and create a list with brief descriptions of each study's findings
+            ```
+            
+            ### **8. Sports & Entertainment**
+            ```
+            URL: https://www.espn.com/nba/standings
+            Query: Extract NBA team standings and create a table with: Team, Wins, Losses, Win Percentage, Conference Position
+            ```
+            
+            ```
+            URL: https://www.imdb.com/chart/top
+            Query: Extract the top 10 movies from IMDb's top 250 list with ratings, year, and brief description
+            ```
+            
+            ### **9. Weather & Environmental Data**
+            ```
+            URL: https://weather.com/weather/today
+            Query: Extract current weather conditions and forecast data. Create a summary with temperature, conditions, and weekly outlook
+            ```
+            
+            ### **10. Real Estate & Property**
+            ```
+            URL: https://www.zillow.com/homes/for_sale
+            Query: Extract property listings with prices, locations, square footage, and key features into a comparison table
+            ```
+            
+            ## 🎯 **Quick Test Samples (Copy & Paste Ready)**
+            
+            ### **Simple Test:**
+            ```
+            URL: https://httpbin.org/html
+            Query: Extract all text content and identify the page structure
+            ```
+            
+            ### **Table Extraction Test:**
+            ```
+            URL: https://www.w3schools.com/html/html_tables.asp
+            Query: Find all HTML tables on this page and convert them to a structured format with proper headers
+            ```
+            
+            ### **Complex Analysis Test:**
+            ```
+            URL: https://www.sec.gov/edgar/browse/?CIK=320193
+            Query: Extract Apple Inc.'s recent SEC filings and create a table with: Filing Date, Document Type, Description
+            ```
+            
+            ### **International Site Test:**
+            ```
+            URL: https://www.bbc.co.uk/weather
+            Query: Extract UK weather information and create a regional breakdown of current conditions
+            ```
+            
+            ## 🔍 **Testing Tips:**
+            
+            1. **Start Simple**: Begin with basic sites like Wikipedia or news sites
+            2. **Test Error Handling**: Try invalid URLs to see error messages
+            3. **Check Timeouts**: Use slow-loading sites to test timeout handling
+            4. **Verify Tables**: Test sites with different table structures
+            5. **Content Variety**: Try different content types (news, data, products)
+            
+            ## 🚨 **Sites That May Have Issues:**
+            - Social media sites (require login)
+            - Sites with heavy JavaScript (may have limited content)
+            - Sites with aggressive bot protection
+            - Password-protected pages
+            
+            ## ✅ **Reliable Test Sites:**
+            - Wikipedia (excellent for tables and structured content)
+            - BBC News (good for text extraction)
+            - Government sites (.gov domains)
+            - W3Schools (great for HTML table testing)
+            - HttpBin (perfect for testing basic functionality)
+            
+            Start with the simpler tests and gradually move to more complex scenarios to fully evaluate your tool's capabilities!
+            """)
+        
+        # Event handlers
+        analyze_btn.click(
+            fn=process_request,
+            inputs=[api_key_input, url_input, query_input],
+            outputs=output,
+            show_progress=True
+        )
+        
+        clear_btn.click(
+            fn=lambda: ("", "", "", ""),
+            outputs=[api_key_input, url_input, query_input, output]
+        )
+    
+    return app
+
+if __name__ == "__main__":
+    # Create and launch the app
+    app = create_interface()
+    
+    # Launch with enhanced configuration
+    app.launch(
+        share=True
+    )