Update index.js

index.js

@@ -13,15 +13,7 @@ app.use(bodyParser.json())
 app.use(express.json({ limit: '500mb' }));
 app.use(cors());
 
-async function extractContentFromUrl(url, browser) {
-    const context = await browser.newContext({
-        userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-        viewport: { width: 1920, height: 1080 },
-        locale: 'en-US',
-        timezoneId: 'America/New_York'
-    });
-    const page = await context.newPage();
-    
+async function extractContentFromUrl(url, page) {
     try {
         page.setDefaultNavigationTimeout(30000);
         page.setDefaultTimeout(30000);
@@ -79,7 +71,7 @@ async function extractContentFromUrl(url, browser) {
                 let totalHeight = 0;
                 const distance = 100;
                 let scrollCount = 0;
-                const maxScrolls = 30; // Limit scrolling
+                const maxScrolls = 30;
                 
                 const timer = setInterval(() => {
                     const scrollHeight = document.body.scrollHeight;
@@ -173,23 +165,6 @@ async function extractContentFromUrl(url, browser) {
             
             const finalMainText = mainText || fallbackText;
             
-            const links = Array.from(document.querySelectorAll('a[href]'))
-                .map(a => ({
-                    text: getTextContent(a),
-                    href: a.href
-                }))
-                .filter(link => link.text && link.href && !link.href.startsWith('javascript:'))
-                .slice(0, 20);
-
-            const images = Array.from(document.querySelectorAll('img[src]'))
-                .filter(img => img.src && !img.src.includes('data:image'))
-                .map(img => ({
-                    src: img.src,
-                    alt: img.alt || '',
-                    title: img.title || ''
-                }))
-                .slice(0, 10);
-            
             const allText = finalMainText || paragraphs.join(' ') || document.body.innerText || '';
             const wordCount = allText.split(/\s+/).filter(word => word.length > 0).length;
 
@@ -202,8 +177,6 @@ async function extractContentFromUrl(url, browser) {
                 },
                 paragraphs,
                 mainText: finalMainText,
-                //links,
-                //images,
                 wordCount,
                 hasContent: wordCount > 50 
             };
@@ -223,8 +196,6 @@ async function extractContentFromUrl(url, browser) {
             error: error.message,
             extractedAt: new Date().toISOString()
         };
-    } finally {
-        await context.close();
     }
 }
 
@@ -277,28 +248,46 @@ app.post('/extract-content', async (req, res) => {
     try {
         browser = await chromium.launch({
             args: [
-                '--incognito', 
-                '--single-process', 
-                '--no-sandbox', 
-                '--no-zygote', 
-                '--no-cache',
-                '--disable-dev-shm-usage',
+                '--no-sandbox',
                 '--disable-setuid-sandbox',
+                '--disable-dev-shm-usage',
                 '--disable-accelerated-2d-canvas',
-                '--disable-gpu'
+                '--disable-gpu',
+                '--disable-blink-features=AutomationControlled',
+                '--disable-web-security',
+                '--disable-features=IsolateOrigins,site-per-process'
             ],
             executablePath: process.env.CHROME_BIN,
             headless: true,
         });
         
-        const concurrencyLimit = 3;
         const results = [];
         
-        for (let i = 0; i < validUrls.length; i += concurrencyLimit) {
-            const batch = validUrls.slice(i, i + concurrencyLimit);
-            const batchPromises = batch.map(url => extractContentFromUrl(url, browser));
-            const batchResults = await Promise.all(batchPromises);
-            results.push(...batchResults);
+        // Process URLs sequentially to avoid browser crashes
+        for (const url of validUrls) {
+            const context = await browser.newContext({
+                userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+                viewport: { width: 1920, height: 1080 },
+                locale: 'en-US',
+                timezoneId: 'America/New_York'
+            });
+            
+            const page = await context.newPage();
+            
+            try {
+                const result = await extractContentFromUrl(url, page);
+                results.push(result);
+            } catch (error) {
+                console.error(`Error processing ${url}:`, error);
+                results.push({
+                    url,
+                    success: false,
+                    error: error.message,
+                    extractedAt: new Date().toISOString()
+                });
+            } finally {
+                await context.close();
+            }
         }
 
         const successCount = results.filter(r => r.success).length;