Update index.js

index.js

@@ -23,20 +23,16 @@ async function extractContentFromUrl(url, browser) {
     const page = await context.newPage();
     
     try {
-        // Set default navigation timeout
         page.setDefaultNavigationTimeout(30000);
         page.setDefaultTimeout(30000);
         
-        // Intercept dan block resource yang tidak perlu
         await page.route('**/*', (route) => {
             const resourceType = route.request().resourceType();
             const url = route.request().url();
             
-            // Block unnecessary resources
             if (['font', 'media', 'websocket'].includes(resourceType)) {
                 route.abort();
             } 
-            // Block tracking and ads
             else if (url.includes('google-analytics') || 
                      url.includes('doubleclick') || 
                      url.includes('facebook') ||
@@ -48,7 +44,6 @@ async function extractContentFromUrl(url, browser) {
             }
         });
 
-        // Try different wait strategies
         try {
             await page.goto(url, {
                 waitUntil: 'domcontentloaded',
@@ -62,28 +57,23 @@ async function extractContentFromUrl(url, browser) {
             });
         }
         
-        // Wait for body to be visible
         try {
             await page.waitForSelector('body', { state: 'visible', timeout: 10000 });
         } catch (e) {
             console.log('Body selector timeout, continuing anyway');
         }
         
-        // Wait a bit for dynamic content
         await page.waitForTimeout(3000);
         
-        // Try to wait for common content selectors
         const contentSelectors = ['article', 'main', '.content', '#content'];
         for (const selector of contentSelectors) {
             try {
                 await page.waitForSelector(selector, { timeout: 5000 });
                 break;
             } catch (e) {
-                // Continue to next selector
             }
         }
         
-        // Gentle scroll untuk trigger lazy loading
         await page.evaluate(() => {
             return new Promise((resolve) => {
                 let totalHeight = 0;
@@ -106,7 +96,6 @@ async function extractContentFromUrl(url, browser) {
             });
         });
         
-        // Wait sebentar setelah scroll
         await page.waitForTimeout(1000);
         
         const content = await page.evaluate(() => {
@@ -114,18 +103,14 @@ async function extractContentFromUrl(url, browser) {
                 return text ? text.replace(/\s+/g, ' ').trim() : '';
             };
             
-            // Helper function untuk mendapatkan text content yang lebih baik
             const getTextContent = (element) => {
                 if (!element) return '';
                 
-                // Clone element untuk manipulasi
                 const clone = element.cloneNode(true);
                 
-                // Remove script dan style tags
                 const scripts = clone.querySelectorAll('script, style, noscript, iframe');
                 scripts.forEach(el => el.remove());
                 
-                // Get text content
                 return cleanText(clone.textContent || clone.innerText || '');
             };
             
@@ -133,7 +118,6 @@ async function extractContentFromUrl(url, browser) {
             const metaDescription = document.querySelector('meta[name="description"]')?.content || 
                                   document.querySelector('meta[property="og:description"]')?.content || '';
             
-            // Improved heading extraction
             const h1Elements = Array.from(document.querySelectorAll('h1'))
                 .map(h1 => getTextContent(h1))
                 .filter(text => text.length > 0);
@@ -142,12 +126,10 @@ async function extractContentFromUrl(url, browser) {
                 .map(h2 => getTextContent(h2))
                 .filter(text => text.length > 0);
             
-            // Improved paragraph extraction
             const paragraphs = Array.from(document.querySelectorAll('p'))
                 .map(p => getTextContent(p))
                 .filter(text => text.length > 20);
             
-            // Try multiple selectors for main content
             const contentSelectors = [
                 'main', 
                 'article', 
@@ -162,7 +144,6 @@ async function extractContentFromUrl(url, browser) {
                 '[itemprop="articleBody"]',
                 '.story-body',
                 '.article-body',
-                // Detik specific selectors
                 '.detail__body-text',
                 '.detail__body',
                 '.itp_bodycontent'
@@ -174,14 +155,12 @@ async function extractContentFromUrl(url, browser) {
                 if (mainContent && getTextContent(mainContent).length > 100) break;
             }
             
-            // Fallback: jika tidak ada main content, ambil dari body
             if (!mainContent) {
                 mainContent = document.body;
             }
             
             const mainText = getTextContent(mainContent);
             
-            // Jika mainText masih kosong, coba ambil semua text dari div yang panjang
             let fallbackText = '';
             if (!mainText || mainText.length < 100) {
                 const allDivs = Array.from(document.querySelectorAll('div'))
@@ -194,7 +173,6 @@ async function extractContentFromUrl(url, browser) {
             
             const finalMainText = mainText || fallbackText;
             
-            // Extract links dengan filter yang lebih baik
             const links = Array.from(document.querySelectorAll('a[href]'))
                 .map(a => ({
                     text: getTextContent(a),
@@ -203,7 +181,6 @@ async function extractContentFromUrl(url, browser) {
                 .filter(link => link.text && link.href && !link.href.startsWith('javascript:'))
                 .slice(0, 20);
 
-            // Extract images dengan filter yang lebih baik
             const images = Array.from(document.querySelectorAll('img[src]'))
                 .filter(img => img.src && !img.src.includes('data:image'))
                 .map(img => ({
@@ -213,7 +190,6 @@ async function extractContentFromUrl(url, browser) {
                 }))
                 .slice(0, 10);
             
-            // Calculate word count
             const allText = finalMainText || paragraphs.join(' ') || document.body.innerText || '';
             const wordCount = allText.split(/\s+/).filter(word => word.length > 0).length;
 
@@ -229,7 +205,7 @@ async function extractContentFromUrl(url, browser) {
                 links,
                 images,
                 wordCount,
-                hasContent: wordCount > 50 // Flag untuk mengecek apakah ada konten
+                hasContent: wordCount > 50 
             };
         });