Update index.js

index.js

@@ -15,40 +15,157 @@ app.use(cors());
 
 async function extractContentFromUrl(url, browser) {
     const context = await browser.newContext({
-        userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+        userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        viewport: { width: 1920, height: 1080 },
+        locale: 'en-US',
+        timezoneId: 'America/New_York'
     });
     const page = await context.newPage();
+    
     try {
+        // Intercept dan block resource yang tidak perlu
+        await page.route('**/*', (route) => {
+            const resourceType = route.request().resourceType();
+            if (['font', 'media'].includes(resourceType)) {
+                route.abort();
+            } else {
+                route.continue();
+            }
+        });
+
         await page.goto(url, {
-            waitUntil: 'domcontentloaded',
+            waitUntil: 'networkidle',
             timeout: 60000
         });
         
-        await page.waitForTimeout(15000);
+        // Wait for content to be visible
+        await page.waitForSelector('body', { state: 'visible', timeout: 30000 });
+        
+        // Scroll untuk trigger lazy loading
+        await page.evaluate(() => {
+            return new Promise((resolve) => {
+                let totalHeight = 0;
+                const distance = 100;
+                const timer = setInterval(() => {
+                    const scrollHeight = document.body.scrollHeight;
+                    window.scrollBy(0, distance);
+                    totalHeight += distance;
+                    
+                    if(totalHeight >= scrollHeight){
+                        clearInterval(timer);
+                        window.scrollTo(0, 0);
+                        resolve();
+                    }
+                }, 100);
+            });
+        });
+        
+        // Wait tambahan setelah scroll
+        await page.waitForTimeout(2000);
         
         const content = await page.evaluate(() => {
             const cleanText = (text) => {
                 return text ? text.replace(/\s+/g, ' ').trim() : '';
             };
             
+            // Helper function untuk mendapatkan text content yang lebih baik
+            const getTextContent = (element) => {
+                if (!element) return '';
+                
+                // Clone element untuk manipulasi
+                const clone = element.cloneNode(true);
+                
+                // Remove script dan style tags
+                const scripts = clone.querySelectorAll('script, style, noscript');
+                scripts.forEach(el => el.remove());
+                
+                // Get text content
+                return cleanText(clone.textContent || clone.innerText || '');
+            };
+            
             const title = document.title || '';
-            const metaDescription = document.querySelector('meta[name="description"]')?.content || '';
-            const h1Elements = Array.from(document.querySelectorAll('h1')).map(h1 => cleanText(h1.innerText));
-            const h2Elements = Array.from(document.querySelectorAll('h2')).map(h2 => cleanText(h2.innerText));
-            const paragraphs = Array.from(document.querySelectorAll('p')).map(p => cleanText(p.innerText)).filter(text => text.length > 20); // Filter paragraf yang terlalu pendek
-            const mainContent = document.querySelector('main, article, .content, #content, .post-content');
-            const mainText = mainContent ? cleanText(mainContent.innerText) : '';
-
-            const links = Array.from(document.querySelectorAll('a[href]')).map(a => ({
-                text: cleanText(a.innerText),
-                href: a.href
-            })).filter(link => link.text && link.href);
+            const metaDescription = document.querySelector('meta[name="description"]')?.content || 
+                                  document.querySelector('meta[property="og:description"]')?.content || '';
+            
+            // Improved heading extraction
+            const h1Elements = Array.from(document.querySelectorAll('h1'))
+                .map(h1 => getTextContent(h1))
+                .filter(text => text.length > 0);
+                
+            const h2Elements = Array.from(document.querySelectorAll('h2'))
+                .map(h2 => getTextContent(h2))
+                .filter(text => text.length > 0);
+            
+            // Improved paragraph extraction
+            const paragraphs = Array.from(document.querySelectorAll('p'))
+                .map(p => getTextContent(p))
+                .filter(text => text.length > 20);
+            
+            // Try multiple selectors for main content
+            const contentSelectors = [
+                'main', 
+                'article', 
+                '[role="main"]',
+                '.content', 
+                '#content', 
+                '.post-content',
+                '.entry-content',
+                '.article-content',
+                '.page-content',
+                '.main-content',
+                '[itemprop="articleBody"]',
+                '.story-body',
+                '.article-body'
+            ];
+            
+            let mainContent = null;
+            for (const selector of contentSelectors) {
+                mainContent = document.querySelector(selector);
+                if (mainContent) break;
+            }
+            
+            // Fallback: jika tidak ada main content, ambil dari body
+            if (!mainContent) {
+                mainContent = document.body;
+            }
+            
+            const mainText = getTextContent(mainContent);
+            
+            // Jika mainText masih kosong, coba ambil semua text dari div yang panjang
+            let fallbackText = '';
+            if (!mainText || mainText.length < 100) {
+                const allDivs = Array.from(document.querySelectorAll('div'))
+                    .map(div => getTextContent(div))
+                    .filter(text => text.length > 200)
+                    .sort((a, b) => b.length - a.length);
+                    
+                fallbackText = allDivs[0] || '';
+            }
+            
+            const finalMainText = mainText || fallbackText;
+            
+            // Extract links dengan filter yang lebih baik
+            const links = Array.from(document.querySelectorAll('a[href]'))
+                .map(a => ({
+                    text: getTextContent(a),
+                    href: a.href
+                }))
+                .filter(link => link.text && link.href && !link.href.startsWith('javascript:'))
+                .slice(0, 20);
 
-            const images = Array.from(document.querySelectorAll('img[src]')).map(img => ({
-                src: img.src,
-                alt: img.alt || '',
-                title: img.title || ''
-            }));
+            // Extract images dengan filter yang lebih baik
+            const images = Array.from(document.querySelectorAll('img[src]'))
+                .filter(img => img.src && !img.src.includes('data:image'))
+                .map(img => ({
+                    src: img.src,
+                    alt: img.alt || '',
+                    title: img.title || ''
+                }))
+                .slice(0, 10);
+            
+            // Calculate word count
+            const allText = finalMainText || paragraphs.join(' ') || document.body.innerText || '';
+            const wordCount = allText.split(/\s+/).filter(word => word.length > 0).length;
 
             return {
                 title,
@@ -58,10 +175,11 @@ async function extractContentFromUrl(url, browser) {
                     h2: h2Elements
                 },
                 paragraphs,
-                mainText,
-                links: links.slice(0, 20),
-                images: images.slice(0, 10),
-                wordCount: (mainText || paragraphs.join(' ')).split(/\s+/).length
+                mainText: finalMainText,
+                links,
+                images,
+                wordCount,
+                hasContent: wordCount > 50 // Flag untuk mengecek apakah ada konten
             };
         });
         
@@ -72,6 +190,7 @@ async function extractContentFromUrl(url, browser) {
             extractedAt: new Date().toISOString()
         };
     } catch (error) {
+        console.error(`Error extracting ${url}:`, error);
         return {
             url,
             success: false,
@@ -131,7 +250,17 @@ app.post('/extract-content', async (req, res) => {
     
     try {
         browser = await chromium.launch({
-            args: ['--incognito', '--single-process', '--no-sandbox', '--no-zygote', '--no-cache'],
+            args: [
+                '--incognito', 
+                '--single-process', 
+                '--no-sandbox', 
+                '--no-zygote', 
+                '--no-cache',
+                '--disable-dev-shm-usage',
+                '--disable-setuid-sandbox',
+                '--disable-accelerated-2d-canvas',
+                '--disable-gpu'
+            ],
             executablePath: process.env.CHROME_BIN,
             headless: true,
         });
@@ -148,6 +277,7 @@ app.post('/extract-content', async (req, res) => {
 
         const successCount = results.filter(r => r.success).length;
         const failCount = results.filter(r => !r.success).length;
+        const emptyContentCount = results.filter(r => r.success && (!r.content.hasContent || r.content.wordCount < 50)).length;
 
         res.json({
             success: true,
@@ -155,7 +285,8 @@ app.post('/extract-content', async (req, res) => {
             statistics: {
                 total: validUrls.length,
                 success: successCount,
-                failed: failCount
+                failed: failCount,
+                emptyContent: emptyContentCount
             },
             results
         });