f1_scraper.py

import os
import asyncio
import argparse
import logging
from datetime import datetime
from urllib.parse import urlparse, urljoin
from typing import List, Dict, Set, Optional, Any
from rich.console import Console
from rich.progress import Progress
from playwright.async_api import async_playwright, TimeoutError
from bs4 import BeautifulSoup
from dotenv import load_dotenv

# Import our custom F1AI class
from f1_ai import F1AI

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
console = Console()

# Load environment variables
load_dotenv()

class F1Scraper:
    def __init__(self, max_pages: int = 100, depth: int = 2, f1_ai: Optional[F1AI] = None):
        """
        Initialize the F1 web scraper.
        
        Args:
            max_pages (int): Maximum number of pages to scrape
            depth (int): Maximum depth for crawling
            f1_ai (F1AI): Optional F1AI instance to use for ingestion
        """
        self.max_pages = max_pages
        self.depth = depth
        self.visited_urls: Set[str] = set()
        self.f1_urls: List[str] = []
        self.f1_ai = f1_ai if f1_ai else F1AI(llm_provider="openrouter")
        
        # Define F1-related keywords to identify relevant pages
        self.f1_keywords = [
            "formula 1", "formula one", "f1", "grand prix", "gp", "race", "racing",
            "driver", "team", "championship", "qualifying", "podium", "ferrari",
            "mercedes", "red bull", "mclaren", "williams", "alpine", "aston martin",
            "haas", "alfa romeo", "alphatauri", "fia", "pirelli", "drs", "pit stop",
            "verstappen", "hamilton", "leclerc", "sainz", "norris", "perez",
            "russell", "alonso", "track", "circuit", "lap", "pole position"
        ]
        
        # Core F1 websites to target
        self.f1_core_sites = [
            "formula1.com",
            "autosport.com",
            "motorsport.com",
            "f1i.com",
            "racefans.net",
            "crash.net/f1",
            "espn.com/f1",
            "bbc.com/sport/formula1",
            "skysports.com/f1"
        ]
    
    def is_f1_related(self, url: str, content: Optional[str] = None) -> bool:
        """Determine if a URL and its content are F1-related."""
        # Check if URL is from a core F1 site
        parsed_url = urlparse(url)
        domain = parsed_url.netloc
        
        for core_site in self.f1_core_sites:
            if core_site in domain:
                return True

        # High-priority paths that are definitely F1-related
        priority_paths = [
            "/racing/", "/drivers/", "/teams/", "/results/",
            "/grands-prix/", "/championship/", "/races/",
            "/season/", "/standings/", "/stats/","/calendar/", 
            "/schedule/"
        ]
        
        # Skip these paths even if they contain F1-related terms
        skip_paths = [
            "/privacy/", "/terms/", "/legal/", "/contact/",
            "/cookie/", "/account/", "/login/", "/register/",
            "/admin/", "/about/", "/careers/", "/press/",
            "/media-centre/", "/corporate/", "/investors/",
            "/f1store", "f1authentincs", "/articles/", "/news/",
            "/blog/", "/videos/", "/photos/", "/gallery/", "/photoshoot/"
        ]

        url_lower = url.lower()
        
        # Check if URL is in skip paths
        if any(path in url_lower for path in skip_paths):
            return False
    
        # Priority paths are always considered F1-related
        if any(path in url_lower for path in priority_paths):
            return True

        # Check URL path for F1 keywords
        url_path = parsed_url.path.lower()
        for keyword in self.f1_keywords:
            if keyword in url_path:
                return True
        
        # If content provided, check for F1 keywords
        if content:
            content_lower = content.lower()
            # Count keyword occurrences to determine relevance
            keyword_count = sum(1 for keyword in self.f1_keywords if keyword in content_lower)
            # If many keywords are found, it's likely F1-related
            if keyword_count >= 3:
                return True
        
        return False
    
    async def extract_links(self, url: str) -> List[str]:
        """Extract links from a webpage."""
        links = []
        try:
            async with async_playwright() as p:
                browser = await p.chromium.launch()
                page = await browser.new_page()
                
                try:
                    await page.goto(url, timeout=30000)
                    html_content = await page.content()
                    soup = BeautifulSoup(html_content, 'html.parser')
                    
                    # Get base domain for domain restriction
                    parsed_url = urlparse(url)
                    base_domain = parsed_url.netloc
                    
                    # Find all links
                    for a_tag in soup.find_all('a', href=True):
                        href = a_tag['href']
                        # Convert relative URLs to absolute
                        if href.startswith('/'):
                            href = urljoin(url, href)
                        
                        # Skip non-http(s) URLs
                        if not href.startswith(('http://', 'https://')):
                            continue
                            
                        # Only include links from formula1.com if it's the default start URL
                        if base_domain == 'www.formula1.com':
                            parsed_href = urlparse(href)
                            if parsed_href.netloc != 'www.formula1.com':
                                continue
                        
                        links.append(href)
                    
                    # Check if content is F1 related before returning
                    text_content = soup.get_text(separator=' ', strip=True)
                    if self.is_f1_related(url, text_content):
                        self.f1_urls.append(url)
                        logger.info(f"✅ F1-related content found: {url}")
                    
                except TimeoutError:
                    logger.error(f"Timeout while loading {url}")
                finally:
                    await browser.close()
            
            return links
        except Exception as e:
            logger.error(f"Error extracting links from {url}: {str(e)}")
            return []
    
    async def crawl(self, start_urls: List[str]) -> List[str]:
        """
        Crawl F1-related websites starting from the provided URLs.
        
        Args:
            start_urls (List[str]): Starting URLs for crawling
            
        Returns:
            List[str]: List of discovered F1-related URLs
        """
        to_visit = start_urls.copy()
        current_depth = 0
        
        with Progress() as progress:
            task = progress.add_task("[green]Crawling F1 websites...", total=self.max_pages)
            
            while to_visit and len(self.visited_urls) < self.max_pages and current_depth <= self.depth:
                current_depth += 1
                next_level = []
                
                for url in to_visit:
                    if url in self.visited_urls:
                        continue
                    
                    self.visited_urls.add(url)
                    progress.update(task, advance=1, description=f"[green]Crawling: {url[:50]}...")
                    
                    links = await self.extract_links(url)
                    next_level.extend([link for link in links if link not in self.visited_urls])
                    
                    # Update progress
                    progress.update(task, completed=len(self.visited_urls), total=self.max_pages)
                    if len(self.visited_urls) >= self.max_pages:
                        break
                
                to_visit = next_level
                logger.info(f"Completed depth {current_depth}, discovered {len(self.f1_urls)} F1-related URLs")
        
        # Deduplicate and return results
        self.f1_urls = list(set(self.f1_urls))
        return self.f1_urls
    
    async def ingest_discovered_urls(self, max_chunks_per_url: int = 50) -> None:
        """
        Ingest discovered F1-related URLs into the RAG system.
        
        Args:
            max_chunks_per_url (int): Maximum chunks to extract per URL
        """
        if not self.f1_urls:
            logger.warning("No F1-related URLs to ingest. Run crawl() first.")
            return
        
        logger.info(f"Ingesting {len(self.f1_urls)} F1-related URLs into RAG system...")
        await self.f1_ai.ingest(self.f1_urls, max_chunks_per_url=max_chunks_per_url)
        logger.info("✅ Ingestion complete!")
    
    def save_urls_to_file(self, filename: str = "f1_urls.txt") -> None:
        """
        Save discovered F1 URLs to a text file.
        
        Args:
            filename (str): Name of the output file
        """
        if not self.f1_urls:
            logger.warning("No F1-related URLs to save. Run crawl() first.")
            return
        
        with open(filename, "w") as f:
            f.write(f"# F1-related URLs discovered on {datetime.now().isoformat()}\n")
            f.write(f"# Total URLs: {len(self.f1_urls)}\n\n")
            for url in self.f1_urls:
                f.write(f"{url}\n")
        
        logger.info(f"✅ Saved {len(self.f1_urls)} URLs to {filename}")

async def main():
    """Main function to run the F1 scraper."""
    parser = argparse.ArgumentParser(description="F1 Web Scraper to discover and ingest F1-related content")
    parser.add_argument("--start-urls", nargs="+", default=["https://www.formula1.com/"], 
                        help="Starting URLs for crawling")
    parser.add_argument("--max-pages", type=int, default=100,
                        help="Maximum number of pages to crawl")
    parser.add_argument("--depth", type=int, default=2,
                        help="Maximum crawl depth")
    parser.add_argument("--ingest", action="store_true",
                        help="Ingest discovered URLs into RAG system")
    parser.add_argument("--max-chunks", type=int, default=50,
                        help="Maximum chunks per URL for ingestion")
    parser.add_argument("--output", type=str, default="f1_urls.txt",
                        help="Output file for discovered URLs")
    parser.add_argument("--llm-provider", choices=["ollama", "openrouter"], default="openrouter",
                        help="Provider for LLM (default: openrouter)")
    
    args = parser.parse_args()
    
    # Initialize F1AI if needed
    f1_ai = None
    if args.ingest:
        f1_ai = F1AI(llm_provider=args.llm_provider)
    
    # Initialize and run the scraper
    scraper = F1Scraper(
        max_pages=args.max_pages,
        depth=args.depth,
        f1_ai=f1_ai
    )
    
    # Crawl to discover F1-related URLs
    console.print("[bold blue]Starting F1 web crawler[/bold blue]")
    discovered_urls = await scraper.crawl(args.start_urls)
    console.print(f"[bold green]Discovered {len(discovered_urls)} F1-related URLs[/bold green]")
    
    # Save URLs to file
    scraper.save_urls_to_file(args.output)
    
    # Ingest if requested
    if args.ingest:
        console.print("[bold yellow]Starting ingestion into RAG system...[/bold yellow]")
        await scraper.ingest_discovered_urls(max_chunks_per_url=args.max_chunks)
        console.print("[bold green]Ingestion complete![/bold green]")

if __name__ == "__main__":
    asyncio.run(main())