Update app.py

app.py

@@ -14,7 +14,7 @@ import sys
 import threading
 from queue import Queue, Empty
 import json
-from concurrent.futures import ThreadPoolExecutor  # Ensure this import is present
+from concurrent.futures import ThreadPoolExecutor
 
 # Import OpenAI library
 import openai
@@ -83,13 +83,10 @@ if not OPENAI_API_KEY:
 openai.api_key = OPENAI_API_KEY
 openai.api_base = "https://api.groq.com/openai/v1"  # Ensure this is the correct base URL for your API
 
-# Initialize global variables for rate limiting
-api_lock = threading.Lock()
-last_api_call_time = 0
-
 # Rate Limiter Configuration
-RPM_LIMIT = 30  # Requests per minute
-TPM_LIMIT = 40000  # Tokens per minute
+RPM_LIMIT = 60  # Requests per minute (adjust based on your API's limit)
+TPM_LIMIT = 60000  # Tokens per minute (adjust based on your API's limit)
+BATCH_SIZE = 5  # Number of bookmarks per batch
 
 # Implementing a Token Bucket Rate Limiter
 class TokenBucket:
@@ -115,7 +112,7 @@ class TokenBucket:
 
     def wait_for_token(self, tokens=1):
         while not self.consume(tokens):
-            time.sleep(0.1)
+            time.sleep(0.05)
 
 # Initialize rate limiters
 rpm_rate = RPM_LIMIT / 60  # tokens per second
@@ -238,137 +235,125 @@ def llm_worker():
     """
     logger.info("LLM worker started.")
     while True:
+        batch = []
         try:
-            bookmark = llm_queue.get(timeout=60)  # Wait for a task
-        except Empty:
-            continue  # No task, continue waiting
-
-        if bookmark is None:
-            logger.info("LLM worker shutting down.")
-            break  # Exit signal
-
-        try:
-            # Rate Limiting
-            rpm_bucket.wait_for_token()
-            # Estimate tokens: prompt + max_tokens
-            # Here, we assume max_tokens=150
-            tpm_bucket.wait_for_token(tokens=150)
-
-            html_content = bookmark.get('html_content', '')
-            soup = BeautifulSoup(html_content, 'html.parser')
-            metadata = get_page_metadata(soup)
-            main_content = extract_main_content(soup)
-
-            # Prepare content for the prompt
-            content_parts = []
-            if metadata['title']:
-                content_parts.append(f"Title: {metadata['title']}")
-            if metadata['description']:
-                content_parts.append(f"Description: {metadata['description']}")
-            if metadata['keywords']:
-                content_parts.append(f"Keywords: {metadata['keywords']}")
-            if main_content:
-                content_parts.append(f"Main Content: {main_content}")
-
-            content_text = '\n'.join(content_parts)
-
-            # Detect insufficient or erroneous content
-            error_keywords = ['Access Denied', 'Security Check', 'Cloudflare', 'captcha', 'unusual traffic']
-            if not content_text or len(content_text.split()) < 50:
-                use_prior_knowledge = True
-                logger.info(f"Content for {bookmark.get('url')} is insufficient. Instructing LLM to use prior knowledge.")
-            elif any(keyword.lower() in content_text.lower() for keyword in error_keywords):
-                use_prior_knowledge = True
-                logger.info(f"Content for {bookmark.get('url')} contains error messages. Instructing LLM to use prior knowledge.")
-            else:
-                use_prior_knowledge = False
-
-            if use_prior_knowledge:
-                prompt = f"""
-You are a knowledgeable assistant with up-to-date information as of 2023.
-URL: {bookmark.get('url')}
-Provide:
-1. A concise summary (max two sentences) about this website.
-2. Assign the most appropriate category from the list below.
-Categories:
-{', '.join([f'"{cat}"' for cat in CATEGORIES])}
-Format:
-Please provide your response in the following JSON format:
-{{
-    "summary": "Your summary here.",
-    "category": "One category from the list."
-}}
-"""
-            else:
-                prompt = f"""
-You are an assistant that creates concise webpage summaries and assigns categories.
-Content:
-{content_text}
-Provide:
-1. A concise summary (max two sentences) focusing on the main topic.
-2. Assign the most appropriate category from the list below.
-Categories:
-{', '.join([f'"{cat}"' for cat in CATEGORIES])}
-Format:
-Please provide your response in the following JSON format:
-{{
-    "summary": "Your summary here.",
-    "category": "One category from the list."
-}}
-"""
-
-            response = openai.ChatCompletion.create(
-                model='llama-3.1-70b-versatile',  # Ensure this model is correct and available
-                messages=[
-                    {"role": "user", "content": prompt}
-                ],
-                max_tokens=150,
-                temperature=0.5,
-            )
+            # Collect bookmarks up to BATCH_SIZE
+            while len(batch) < BATCH_SIZE:
+                bookmark = llm_queue.get(timeout=1)
+                if bookmark is None:
+                    # Shutdown signal
+                    logger.info("LLM worker shutting down.")
+                    return
+                if not bookmark.get('dead_link') and not bookmark.get('slow_link'):
+                    batch.append(bookmark)
+                else:
+                    # Skip processing for dead or slow links
+                    bookmark['summary'] = 'No summary available.'
+                    bookmark['category'] = 'Uncategorized'
+                    llm_queue.task_done()
 
-            content = response['choices'][0]['message']['content'].strip()
-            if not content:
-                raise ValueError("Empty response received from the model.")
+        except Empty:
+            pass  # No more bookmarks at the moment
 
-            # Parse JSON response
+        if batch:
             try:
-                json_response = json.loads(content)
-                summary = json_response.get('summary', '').strip()
-                category = json_response.get('category', '').strip()
-
-                # Validate and assign
-                if not summary:
-                    summary = metadata.get('description') or metadata.get('title') or 'No summary available.'
-                bookmark['summary'] = summary
+                # Rate Limiting
+                rpm_bucket.wait_for_token()
+                # Estimate tokens: prompt + max_tokens
+                # Here, we assume max_tokens=150 per bookmark
+                total_tokens = 150 * len(batch)
+                tpm_bucket.wait_for_token(tokens=total_tokens)
+
+                # Prepare prompt
+                prompt = "You are an assistant that creates concise webpage summaries and assigns categories.\n\n"
+                prompt += "Provide summaries and categories for the following bookmarks:\n\n"
+
+                for idx, bookmark in enumerate(batch, 1):
+                    prompt += f"Bookmark {idx}:\nURL: {bookmark['url']}\nTitle: {bookmark['title']}\n\n"
+
+                prompt += f"Categories:\n{', '.join([f'\"{cat}\"' for cat in CATEGORIES])}\n\n"
+                prompt += "Format your response as a JSON object where each key is the bookmark URL and the value is another JSON object containing 'summary' and 'category'.\n\n"
+                prompt += "Example:\n"
+                prompt += "{\n"
+                prompt += "  \"https://example.com\": {\n"
+                prompt += "    \"summary\": \"This is an example summary.\",\n"
+                prompt += "    \"category\": \"Technology\"\n"
+                prompt += "  }\n"
+                prompt += "}\n\n"
+                prompt += "Now, provide the summaries and categories for the bookmarks listed above."
+
+                response = openai.ChatCompletion.create(
+                    model='llama-3.1-70b-versatile',  # Ensure this model is correct and available
+                    messages=[
+                        {"role": "user", "content": prompt}
+                    ],
+                    max_tokens=150 * len(batch),
+                    temperature=0.5,
+                )
 
-                if category in CATEGORIES:
-                    bookmark['category'] = category
-                else:
-                    # Fallback to keyword-based categorization
-                    bookmark['category'] = categorize_based_on_summary(summary, bookmark['url'])
-
-            except json.JSONDecodeError:
-                logger.error(f"Failed to parse JSON response for {bookmark.get('url')}. Using fallback methods.")
-                # Fallback methods
-                bookmark['summary'] = metadata.get('description') or metadata.get('title') or 'No summary available.'
-                bookmark['category'] = categorize_based_on_summary(bookmark['summary'], bookmark['url'])
-
-            # Additional keyword-based validation
-            bookmark['category'] = validate_category(bookmark)
-
-            logger.info("Successfully generated summary and assigned category")
-        except openai.error.RateLimitError as e:
-            logger.warning(f"LLM Rate limit reached while processing {bookmark.get('url')}. Retrying later...")
-            # Re-enqueue the bookmark for retry
-            llm_queue.put(bookmark)
-            time.sleep(60)  # Wait before retrying
-        except Exception as e:
-            logger.error(f"Error generating summary and assigning category for {bookmark.get('url')}: {e}", exc_info=True)
-            # Assign default values on failure
-            bookmark['summary'] = 'No summary available.'
-            bookmark['category'] = 'Uncategorized'
-        finally:
-            llm_queue.task_done()
+                content = response['choices'][0]['message']['content'].strip()
+                if not content:
+                    raise ValueError("Empty response received from the model.")
+
+                # Parse JSON response
+                try:
+                    json_response = json.loads(content)
+                    for bookmark in batch:
+                        url = bookmark['url']
+                        if url in json_response:
+                            summary = json_response[url].get('summary', '').strip()
+                            category = json_response[url].get('category', '').strip()
+
+                            if not summary:
+                                summary = 'No summary available.'
+                            bookmark['summary'] = summary
+
+                            if category in CATEGORIES:
+                                bookmark['category'] = category
+                            else:
+                                # Fallback to keyword-based categorization
+                                bookmark['category'] = categorize_based_on_summary(summary, url)
+                        else:
+                            logger.warning(f"No data returned for {url}. Using fallback methods.")
+                            bookmark['summary'] = 'No summary available.'
+                            bookmark['category'] = 'Uncategorized'
+
+                        # Additional keyword-based validation
+                        bookmark['category'] = validate_category(bookmark)
+
+                        logger.info(f"Processed bookmark: {url}")
+
+                except json.JSONDecodeError:
+                    logger.error("Failed to parse JSON response from LLM. Using fallback methods.")
+                    for bookmark in batch:
+                        bookmark['summary'] = 'No summary available.'
+                        bookmark['category'] = categorize_based_on_summary(bookmark.get('summary', ''), bookmark['url'])
+                        bookmark['category'] = validate_category(bookmark)
+
+                except Exception as e:
+                    logger.error(f"Error processing LLM response: {e}", exc_info=True)
+                    for bookmark in batch:
+                        bookmark['summary'] = 'No summary available.'
+                        bookmark['category'] = 'Uncategorized'
+
+            except openai.error.RateLimitError as e:
+                logger.warning(f"LLM Rate limit reached. Retrying after 60 seconds.")
+                # Re-enqueue the entire batch for retry
+                for bookmark in batch:
+                    llm_queue.put(bookmark)
+                time.sleep(60)  # Wait before retrying
+                continue  # Skip the rest and retry
+
+            except Exception as e:
+                logger.error(f"Error during LLM processing: {e}", exc_info=True)
+                for bookmark in batch:
+                    bookmark['summary'] = 'No summary available.'
+                    bookmark['category'] = 'Uncategorized'
+
+            finally:
+                # Mark all bookmarks in the batch as done
+                for _ in batch:
+                    llm_queue.task_done()
 
 def generate_summary_and_assign_category(bookmark):
     """
@@ -704,7 +689,9 @@ def chatbot_response(user_query, chat_history):
 
         # Rate Limiting
         rpm_bucket.wait_for_token()
-        tpm_bucket.wait_for_token(tokens=300)  # Assuming max_tokens=300 per request
+        # Estimate tokens: prompt + max_tokens
+        # Here, we assume max_tokens=300 per chatbot response
+        tpm_bucket.wait_for_token(tokens=300)
 
         query_vector = embedding_model.encode([user_query]).astype('float32')
         k = 5