Update app.py

app.py

@@ -19,6 +19,10 @@ import threading
 # Import OpenAI library
 import openai
 
+# Suppress only the single warning from urllib3 needed.
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
 # Set up logging to output to the console
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -295,6 +299,7 @@ Category: [One category]
             time.sleep(wait_time)
         except Exception as e:
             logger.error(f"Error generating summary and assigning category: {e}", exc_info=True)
+            # Ensure 'summary' is always set, even on failure
             bookmark['summary'] = 'No summary available.'
             bookmark['category'] = 'Uncategorized'
             break  # Exit the retry loop on other exceptions
@@ -337,7 +342,7 @@ def fetch_url_info(bookmark):
             'User-Agent': 'Mozilla/5.0',
             'Accept-Language': 'en-US,en;q=0.9',
         }
-        response = requests.get(url, headers=headers, timeout=5, verify=False, allow_redirects=True)
+        response = requests.get(url, headers=headers, timeout=5, verify=True, allow_redirects=True)  # Set verify=True
         bookmark['etag'] = response.headers.get('ETag', 'N/A')
         bookmark['status_code'] = response.status_code
 
@@ -357,6 +362,13 @@ def fetch_url_info(bookmark):
             bookmark['description'] = ''
             logger.info(f"Fetched information for {url}")
 
+    except requests.exceptions.SSLError as e:
+        bookmark['dead_link'] = True
+        bookmark['etag'] = 'N/A'
+        bookmark['status_code'] = 'SSL Error'
+        bookmark['description'] = ''
+        bookmark['html_content'] = ''
+        logger.error(f"SSL error fetching URL info for {url}: {e}", exc_info=True)
     except requests.exceptions.Timeout:
         bookmark['dead_link'] = False  # Mark as 'Unknown' instead of 'Dead'
         bookmark['etag'] = 'N/A'
@@ -390,7 +402,15 @@ def vectorize_and_index(bookmarks_list):
     global faiss_index
     logger.info("Vectorizing summaries and building FAISS index")
     try:
-        summaries = [bookmark['summary'] for bookmark in bookmarks_list]
+        # Use .get('summary', '') to avoid KeyError
+        summaries = [bookmark.get('summary', '') for bookmark in bookmarks_list]
+        
+        # Check for any empty summaries and log them
+        for i, summary in enumerate(summaries):
+            if not summary:
+                logger.warning(f"Bookmark at index {i} is missing a summary.")
+                summaries[i] = 'No summary available.'
+
         embeddings = get_embedding_model().encode(summaries).astype('float32')
         dimension = embeddings.shape[1]
         if faiss_index is None:
@@ -509,6 +529,13 @@ def process_uploaded_file(file, state_bookmarks):
     with ThreadPoolExecutor(max_workers=3) as executor:  # Adjusted max_workers
         executor.map(generate_summary_and_assign_category, bookmarks)
 
+    # Log bookmarks to verify 'summary' and 'category' presence
+    for idx, bookmark in enumerate(bookmarks):
+        if 'summary' not in bookmark or 'category' not in bookmark:
+            logger.error(f"Bookmark at index {idx} is missing 'summary' or 'category': {bookmark}")
+        else:
+            logger.debug(f"Bookmark {idx} processed with summary and category.")
+
     try:
         vectorize_and_index(bookmarks)
     except Exception as e: