Update app.py

app.py

@@ -78,22 +78,33 @@ def extract_main_content(soup):
     """
     Extract the main content from a webpage while filtering out boilerplate content.
     """
+    if not soup:
+        return ""
+        
     # Remove script and style elements
     for element in soup(['script', 'style', 'header', 'footer', 'nav', 'ads', 'sidebar']):
         element.decompose()
     
-    # Get text from specific content tags first
-    main_content_tags = soup.find_all(['article', 'main', 'div.content', 'div.post'])
+    # First try to find content in main content areas
+    main_content_tags = soup.find_all(['article', 'main', 'div.content', 'div.post', 'div.entry-content'])
     if main_content_tags:
         content = ' '.join([tag.get_text(strip=True, separator=' ') for tag in main_content_tags])
     else:
-        # Fallback to body content
-        content = soup.body.get_text(strip=True, separator=' ') if soup.body else soup.get_text(strip=True, separator=' ')
+        # Try to find content in <p> tags
+        p_tags = soup.find_all('p')
+        if p_tags:
+            content = ' '.join([p.get_text(strip=True, separator=' ') for p in p_tags])
+        else:
+            # Fallback to body content
+            content = soup.body.get_text(strip=True, separator=' ') if soup.body else soup.get_text(strip=True, separator=' ')
     
     # Clean up the text
     content = ' '.join(content.split())
-    # Limit content length to avoid token limits
-    return content[:3000]
+    content = re.sub(r'\s+', ' ', content)  # Remove multiple spaces
+    content = re.sub(r'[\n\r\t]', ' ', content)  # Remove newlines and tabs
+    
+    # Limit content length to avoid token limits (adjust as needed)
+    return content[:5000]
 
 def get_page_metadata(soup):
     """
@@ -105,14 +116,20 @@ def get_page_metadata(soup):
         'keywords': ''
     }
     
+    if not soup:
+        return metadata
+        
     # Get title
     title_tag = soup.find('title')
-    if title_tag:
+    if title_tag and title_tag.string:
         metadata['title'] = title_tag.string.strip()
     
-    # Get meta description
-    meta_desc = soup.find('meta', attrs={'name': 'description'}) or \
-                soup.find('meta', attrs={'property': 'og:description'})
+    # Get meta description (try multiple variants)
+    meta_desc = (
+        soup.find('meta', attrs={'name': 'description'}) or
+        soup.find('meta', attrs={'property': 'og:description'}) or
+        soup.find('meta', attrs={'name': 'twitter:description'})
+    )
     if meta_desc:
         metadata['description'] = meta_desc.get('content', '').strip()
     
@@ -121,6 +138,12 @@ def get_page_metadata(soup):
     if meta_keywords:
         metadata['keywords'] = meta_keywords.get('content', '').strip()
     
+    # Get OG title if main title is empty
+    if not metadata['title']:
+        og_title = soup.find('meta', attrs={'property': 'og:title'})
+        if og_title:
+            metadata['title'] = og_title.get('content', '').strip()
+    
     return metadata
 
 def generate_summary(bookmark):
@@ -133,37 +156,39 @@ def generate_summary(bookmark):
         # Get the HTML soup object from the bookmark if it exists
         soup = BeautifulSoup(bookmark.get('html_content', ''), 'html.parser')
         
-        # Step 1: Try to get description from metadata
+        # Step 1: Extract all available information
         metadata = get_page_metadata(soup)
-        if metadata['description']:
-            logger.info("Using meta description for summary")
-            bookmark['summary'] = metadata['description']
-            return bookmark
+        main_content = extract_main_content(soup)
         
-        # Step 2: If no description, extract main content
-        content = extract_main_content(soup)
-        if not content:
-            logger.warning("No content extracted from page")
-            # Fallback to title if available
+        # Step 2: Generate summary using LLM with all available content
+        try:
+            # Prepare comprehensive context for LLM
+            available_content = []
             if metadata['title']:
-                bookmark['summary'] = f"Page title: {metadata['title']}"
-                return bookmark
+                available_content.append(f"Title: {metadata['title']}")
+            if metadata['description']:
+                available_content.append(f"Description: {metadata['description']}")
+            if metadata['keywords']:
+                available_content.append(f"Keywords: {metadata['keywords']}")
+            if main_content:
+                available_content.append(f"Main Content: {main_content}")
             
-            bookmark['summary'] = bookmark.get('title', 'No summary available.')
-            return bookmark
-        
-        # Step 3: Generate summary using LLM
-        try:
-            # Prepare context for LLM
+            if not available_content:
+                logger.warning("No content available for summary generation")
+                bookmark['summary'] = bookmark.get('title', 'No summary available.')
+                return bookmark
+                
             prompt = f"""
-            Webpage Title: {metadata['title']}
-            Keywords: {metadata['keywords']}
+            Analyze and summarize this webpage based on the following information:
+            
+            {' | '.join(available_content)}
             
-            Content:
-            {content}
+            Please provide a concise summary (2-3 sentences) focusing on:
+            1. The main purpose or topic of the page
+            2. Key information or features
+            3. Target audience or use case (if apparent)
             
-            Please provide a concise summary (2-3 sentences) of this webpage's main content.
-            Focus on what the page is about and its key information. Be factual and objective.
+            Be factual and objective.
             """
             
             response = openai.ChatCompletion.create(
@@ -183,13 +208,22 @@ def generate_summary(bookmark):
             
         except Exception as e:
             logger.error(f"Error generating LLM summary: {e}")
-            # Fallback to extracted content
-            bookmark['summary'] = ' '.join(content.split()[:50]) + '...'
+            # Fallback mechanisms in order of preference
+            if metadata['description']:
+                logger.info("Falling back to meta description")
+                bookmark['summary'] = metadata['description']
+            elif main_content:
+                logger.info("Falling back to truncated main content")
+                bookmark['summary'] = ' '.join(main_content.split()[:50]) + '...'
+            elif metadata['title']:
+                logger.info("Falling back to title")
+                bookmark['summary'] = metadata['title']
+            else:
+                bookmark['summary'] = bookmark.get('title', 'No summary available.')
             return bookmark
             
     except Exception as e:
         logger.error(f"Error in generate_summary: {e}")
-        # Final fallback
         bookmark['summary'] = bookmark.get('title', 'No summary available.')
         return bookmark
 
@@ -223,18 +257,22 @@ async def fetch_url_info(session, bookmark):
 
     try:
         logger.info(f"Fetching URL info for: {url}")
-        async with session.get(url, timeout=10) as response:
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+        }
+        async with session.get(url, timeout=10, headers=headers) as response:
             bookmark['etag'] = response.headers.get('ETag', 'N/A')
             bookmark['status_code'] = response.status
 
             if response.status >= 400:
                 bookmark['dead_link'] = True
                 bookmark['description'] = ''
+                bookmark['html_content'] = ''
                 logger.warning(f"Dead link detected: {url} with status {response.status}")
             else:
                 bookmark['dead_link'] = False
                 content = await response.text()
-                bookmark['html_content'] = content
+                bookmark['html_content'] = content  # Store full HTML for summary generation
                 bookmark['description'] = ''  # Will be set by generate_summary function
                 logger.info(f"Fetched information for {url}")
     except Exception as e:
@@ -260,7 +298,9 @@ async def process_bookmarks_async(bookmarks_list):
     """
     logger.info("Processing bookmarks asynchronously")
     try:
-        async with aiohttp.ClientSession() as session:
+        connector = aiohttp.TCPConnector(limit=5)  # Limit concurrent connections
+        timeout = aiohttp.ClientTimeout(total=30)  # Set timeout
+        async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
             tasks = []
             for bookmark in bookmarks_list:
                 task = asyncio.ensure_future(fetch_url_info(session, bookmark))
@@ -384,30 +424,30 @@ def process_uploaded_file(file):
     
     if file is None:
         logger.warning("No file uploaded")
-        return "Please upload a bookmarks HTML file.", ''
+        return "Please upload a bookmarks HTML file.", '', gr.update(choices=[]), display_bookmarks()
 
     try:
         file_content = file.decode('utf-8')
     except UnicodeDecodeError as e:
         logger.error(f"Error decoding the file: {e}")
-        return "Error decoding the file. Please ensure it's a valid HTML file.", ''
+        return "Error decoding the file. Please ensure it's a valid HTML file.", '', gr.update(choices=[]), display_bookmarks()
 
     try:
         bookmarks = parse_bookmarks(file_content)
     except Exception as e:
         logger.error(f"Error parsing bookmarks: {e}")
-        return "Error parsing the bookmarks HTML file.", ''
+        return "Error parsing the bookmarks HTML file.", '', gr.update(choices=[]), display_bookmarks()
 
     if not bookmarks:
         logger.warning("No bookmarks found in the uploaded file")
-        return "No bookmarks found in the uploaded file.", ''
+        return "No bookmarks found in the uploaded file.", '', gr.update(choices=[]), display_bookmarks()
 
     # Asynchronously fetch bookmark info
     try:
         asyncio.run(process_bookmarks_async(bookmarks))
     except Exception as e:
         logger.error(f"Error processing bookmarks asynchronously: {e}")
-        return "Error processing bookmarks.", ''
+        return "Error processing bookmarks.", '', gr.update(choices=[]), display_bookmarks()
 
     # Generate summaries and assign categories
     for bookmark in bookmarks:
@@ -418,21 +458,17 @@ def process_uploaded_file(file):
         faiss_index, embeddings = vectorize_and_index(bookmarks)
     except Exception as e:
         logger.error(f"Error building FAISS index: {e}")
-        return "Error building search index.", ''
+        return "Error building search index.", '', gr.update(choices=[]), display_bookmarks()
 
     message = f"✅ Successfully processed {len(bookmarks)} bookmarks."
     logger.info(message)
+    
+    # Generate displays and updates
     bookmark_html = display_bookmarks()
-
-    return message, bookmark_html
-
-def update_bookmark_selector():
-    """
-    Update the bookmark selector choices for the Manage Bookmarks tab.
-    """
     choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" 
               for i, bookmark in enumerate(bookmarks)]
-    return gr.update(choices=choices, value=[])
+
+    return message, bookmark_html, gr.update(choices=choices), bookmark_html
 
 def delete_selected_bookmarks(selected_indices):
     """
@@ -440,7 +476,7 @@ def delete_selected_bookmarks(selected_indices):
     """
     global bookmarks, faiss_index
     if not selected_indices:
-        return "⚠️ No bookmarks selected.", gr.update(choices=[]), ''
+        return "⚠️ No bookmarks selected.", gr.update(choices=[]), display_bookmarks()
 
     indices = [int(s.split('.')[0])-1 for s in selected_indices]
     indices = sorted(indices, reverse=True)
@@ -457,19 +493,20 @@ def delete_selected_bookmarks(selected_indices):
     message = "🗑️ Selected bookmarks deleted successfully."
     logger.info(message)
     
-    bookmark_selector_update = update_bookmark_selector()
-    bookmarks_html = display_bookmarks()
+    # Update choices and display
+    choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" 
+              for i, bookmark in enumerate(bookmarks)]
     
-    return message, bookmark_selector_update, bookmarks_html
+    return message, gr.update(choices=choices), display_bookmarks()
 
 def edit_selected_bookmarks_category(selected_indices, new_category):
     """
     Edit category of selected bookmarks.
     """
     if not selected_indices:
-        return "⚠️ No bookmarks selected.", gr.update(choices=[]), ''
+        return "⚠️ No bookmarks selected.", gr.update(choices=[]), display_bookmarks()
     if not new_category:
-        return "⚠️ No new category selected.", gr.update(choices=[]), ''
+        return "⚠️ No new category selected.", gr.update(choices=[]), display_bookmarks()
 
     indices = [int(s.split('.')[0])-1 for s in selected_indices]
     for idx in indices:
@@ -480,10 +517,11 @@ def edit_selected_bookmarks_category(selected_indices, new_category):
     message = "✏️ Category updated for selected bookmarks."
     logger.info(message)
     
-    bookmark_selector_update = update_bookmark_selector()
-    bookmarks_html = display_bookmarks()
+    # Update choices and display
+    choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" 
+              for i, bookmark in enumerate(bookmarks)]
     
-    return message, bookmark_selector_update, bookmarks_html
+    return message, gr.update(choices=choices), display_bookmarks()
 
 def export_bookmarks():
     """
@@ -604,12 +642,6 @@ def build_app():
                 output_text = gr.Textbox(label="✅ Output", interactive=False)
                 bookmark_display = gr.HTML(label="📄 Processed Bookmarks")
 
-                process_button.click(
-                    process_uploaded_file,
-                    inputs=upload,
-                    outputs=[output_text, bookmark_display]
-                )
-
             # Chat with Bookmarks Tab
             with gr.Tab("Chat with Bookmarks"):
                 gr.Markdown("""
@@ -625,17 +657,10 @@ def build_app():
                 chat_button = gr.Button("📨 Send")
                 chat_output = gr.Textbox(label="💬 Response", interactive=False)
 
-                chat_button.click(
-                    chatbot_response,
-                    inputs=user_input,
-                    outputs=chat_output
-                )
-
             # Manage Bookmarks Tab
             with gr.Tab("Manage Bookmarks"):
                 gr.Markdown("""
                 ## 🛠️ **Manage Bookmarks**
-
                 Select bookmarks to delete or edit their categories.
                 """)
 
@@ -649,31 +674,44 @@ def build_app():
                     choices=CATEGORIES,
                     value="Uncategorized"
                 )
+                bookmark_display_manage = gr.HTML(label="📄 Bookmarks")
 
                 with gr.Row():
                     delete_button = gr.Button("🗑️ Delete Selected")
                     edit_category_button = gr.Button("✏️ Edit Category")
                     export_button = gr.Button("💾 Export")
-
-                bookmark_display_manage = gr.HTML(label="📄 Bookmarks")
+                
                 download_link = gr.HTML(label="📥 Download")
 
-                delete_button.click(
-                    delete_selected_bookmarks,
-                    inputs=bookmark_selector,
-                    outputs=[manage_output, bookmark_selector, bookmark_display_manage]
-                )
+            # Set up event handlers
+            process_button.click(
+                process_uploaded_file,
+                inputs=upload,
+                outputs=[output_text, bookmark_display, bookmark_selector, bookmark_display_manage]
+            )
 
-                edit_category_button.click(
-                    edit_selected_bookmarks_category,
-                    inputs=[bookmark_selector, new_category],
-                    outputs=[manage_output, bookmark_selector, bookmark_display_manage]
-                )
+            chat_button.click(
+                chatbot_response,
+                inputs=user_input,
+                outputs=chat_output
+            )
 
-                export_button.click(
-                    export_bookmarks,
-                    outputs=download_link
-                )
+            delete_button.click(
+                delete_selected_bookmarks,
+                inputs=bookmark_selector,
+                outputs=[manage_output, bookmark_selector, bookmark_display_manage]
+            )
+
+            edit_category_button.click(
+                edit_selected_bookmarks_category,
+                inputs=[bookmark_selector, new_category],
+                outputs=[manage_output, bookmark_selector, bookmark_display_manage]
+            )
+
+            export_button.click(
+                export_bookmarks,
+                outputs=download_link
+            )
 
         logger.info("Launching Gradio app")
         demo.launch(debug=True)