Update main.py

main.py

@@ -1,126 +1,126 @@
-from fastapi import FastAPI, HTTPException, Request
-from pydantic import HttpUrl
-from playwright.async_api import async_playwright
-from urllib.parse import urljoin, urlparse
-import logging
-from fastapi.responses import JSONResponse
-from fastapi.templating import Jinja2Templates
-from fastapi.staticfiles import StaticFiles
-from typing import List, Dict
-import asyncio
-
-# Set up logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-
-app = FastAPI(title="Website Scraper API (Enhanced for Images)")
-
-# Mount static files
-app.mount("/static", StaticFiles(directory="static"), name="static")
-
-# Set up Jinja2 templates
-templates = Jinja2Templates(directory="templates")
-
-# Maximum number of pages to scrape
-MAX_PAGES = 20
-
-async def scrape_page(url: str, visited: set, base_domain: str) -> tuple[Dict, set]:
-    """Scrape a single page for text, images, and links using Playwright."""
-    try:
-        async with async_playwright() as p:
-            browser = await p.chromium.launch(headless=True)
-            context = await browser.new_context(
-                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
-                viewport={"width": 1280, "height": 720}
-            )
-            page = await context.new_page()
-            
-            # Navigate and wait for content
-            await page.goto(url, wait_until="networkidle", timeout=30000)
-            
-            # Scroll to trigger lazy-loaded images
-            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
-            await page.wait_for_timeout(2000)  # Wait for lazy-loaded content
-            
-            # Extract text content
-            text_content = await page.evaluate(
-                """() => document.body.innerText"""
-            )
-            text_content = ' '.join(text_content.split()) if text_content else ""
-
-            # Extract images from src, data-src, and srcset
-            images = await page.evaluate(
-                """() => {
-                    const imgElements = document.querySelectorAll('img');
-                    const imgUrls = new Set();
-                    imgElements.forEach(img => {
-                        if (img.src) imgUrls.add(img.src);
-                        if (img.dataset.src) imgUrls.add(img.dataset.src);
-                        if (img.srcset) {
-                            img.srcset.split(',').forEach(src => {
-                                const url = src.trim().split(' ')[0];
-                                if (url) imgUrls.add(url);
-                            });
-                        }
-                    });
-                    return Array.from(imgUrls);
-                }"""
-            )
-            images = [urljoin(url, img) for img in images if img]
-
-            # Extract links
-            links = await page.evaluate(
-                """() => Array.from(document.querySelectorAll('a')).map(a => a.href)"""
-            )
-            links = set(urljoin(url, link) for link in links if urlparse(urljoin(url, link)).netloc == base_domain and urljoin(url, link) not in visited)
-
-            await browser.close()
-
-        page_data = {
-            "url": url,
-            "text": text_content,
-            "images": images
-        }
-        return page_data, links
-
-    except Exception as e:
-        logging.error(f"Error scraping {url}: {e}")
-        return {}, set()
-
-@app.get("/scrape")
-async def crawl_website(url: HttpUrl):
-    """Crawl the website starting from the given URL and return scraped data for up to 10 pages as JSON."""
-    try:
-        visited = set()
-        to_visit = {str(url)}
-        base_domain = urlparse(str(url)).netloc
-        results = []
-
-        while to_visit and len(visited) < MAX_PAGES:
-            current_url = to_visit.pop()
-            if current_url in visited:
-                continue
-
-            logging.info(f"Scraping: {current_url}")
-            visited.add(current_url)
-
-            page_data, new_links = await scrape_page(current_url, visited, base_domain)
-            if page_data:
-                results.append(page_data)
-            to_visit.update(new_links)
-            
-            # Small delay to avoid overwhelming the server
-            await asyncio.sleep(0.5)
-
-        return JSONResponse(content={"pages": results})
-
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Scraping failed: {str(e)}")
-
-@app.get("/")
-async def serve_home(request: Request):
-    """Serve the frontend HTML page."""
-    return templates.TemplateResponse("index.html", {"request": request})
-
-if __name__ == "__main__":
-    import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=8001)
+from fastapi import FastAPI, HTTPException, Request
+from pydantic import HttpUrl
+from playwright.async_api import async_playwright
+from urllib.parse import urljoin, urlparse
+import logging
+from fastapi.responses import JSONResponse
+from fastapi.templating import Jinja2Templates
+from fastapi.staticfiles import StaticFiles
+from typing import List, Dict
+import asyncio
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+app = FastAPI(title="Website Scraper API (Enhanced for Images)")
+
+# Mount static files
+app.mount("/static", StaticFiles(directory="static"), name="static")
+
+# Set up Jinja2 templates
+templates = Jinja2Templates(directory="templates")
+
+# Maximum number of pages to scrape
+MAX_PAGES = 20
+
+async def scrape_page(url: str, visited: set, base_domain: str) -> tuple[Dict, set]:
+    """Scrape a single page for text, images, and links using Playwright."""
+    try:
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context(
+                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
+                viewport={"width": 1280, "height": 720}
+            )
+            page = await context.new_page()
+            
+            # Navigate and wait for content
+            await page.goto(url, wait_until="networkidle", timeout=30000)
+            
+            # Scroll to trigger lazy-loaded images
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            await page.wait_for_timeout(2000)  # Wait for lazy-loaded content
+            
+            # Extract text content
+            text_content = await page.evaluate(
+                """() => document.body.innerText"""
+            )
+            text_content = ' '.join(text_content.split()) if text_content else ""
+
+            # Extract images from src, data-src, and srcset
+            images = await page.evaluate(
+                """() => {
+                    const imgElements = document.querySelectorAll('img');
+                    const imgUrls = new Set();
+                    imgElements.forEach(img => {
+                        if (img.src) imgUrls.add(img.src);
+                        if (img.dataset.src) imgUrls.add(img.dataset.src);
+                        if (img.srcset) {
+                            img.srcset.split(',').forEach(src => {
+                                const url = src.trim().split(' ')[0];
+                                if (url) imgUrls.add(url);
+                            });
+                        }
+                    });
+                    return Array.from(imgUrls);
+                }"""
+            )
+            images = [urljoin(url, img) for img in images if img]
+
+            # Extract links
+            links = await page.evaluate(
+                """() => Array.from(document.querySelectorAll('a')).map(a => a.href)"""
+            )
+            links = set(urljoin(url, link) for link in links if urlparse(urljoin(url, link)).netloc == base_domain and urljoin(url, link) not in visited)
+
+            await browser.close()
+
+        page_data = {
+            "url": url,
+            "text": text_content,
+            "images": images
+        }
+        return page_data, links
+
+    except Exception as e:
+        logging.error(f"Error scraping {url}: {e}")
+        return {}, set()
+
+@app.get("/scrape")
+async def crawl_website(url: HttpUrl):
+    """Crawl the website starting from the given URL and return scraped data for up to 10 pages as JSON."""
+    try:
+        visited = set()
+        to_visit = {str(url)}
+        base_domain = urlparse(str(url)).netloc
+        results = []
+
+        while to_visit and len(visited) < MAX_PAGES:
+            current_url = to_visit.pop()
+            if current_url in visited:
+                continue
+
+            logging.info(f"Scraping: {current_url}")
+            visited.add(current_url)
+
+            page_data, new_links = await scrape_page(current_url, visited, base_domain)
+            if page_data:
+                results.append(page_data)
+            to_visit.update(new_links)
+            
+            # Small delay to avoid overwhelming the server
+            await asyncio.sleep(0.5)
+
+        return JSONResponse(content={"pages": results})
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Scraping failed: {str(e)}")
+
+@app.get("/")
+async def serve_home(request: Request):
+    """Serve the frontend HTML page."""
+    return templates.TemplateResponse("index.html", {"request": request})
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)