Update app.py

app.py

@@ -15,6 +15,7 @@ import sys
 import concurrent.futures
 from concurrent.futures import ThreadPoolExecutor
 import threading
+from ratelimiter import RateLimiter  # Optional
 
 # Import OpenAI library
 import openai
@@ -80,7 +81,10 @@ if not GROQ_API_KEY:
     logger.error("GROQ_API_KEY environment variable not set.")
 
 openai.api_key = GROQ_API_KEY
-openai.api_base = "https://api.groq.com/openai/v1"
+openai.api_base = "https://api.groq.com/openai/v1"  # Ensure this is the correct base URL
+
+# Initialize rate limiter (optional, adjust based on rate limits)
+llm_rate_limiter = RateLimiter(max_calls=20, period=60)  # Example: 20 calls per minute
 
 # Global variables for models to enable lazy loading
 embedding_model = None
@@ -168,21 +172,20 @@ def generate_summary_and_assign_category(bookmark):
     """
     logger.info(f"Generating summary and assigning category for bookmark: {bookmark.get('url')}")
 
-    max_retries = 3
+    max_retries = 5
     retry_count = 0
+    base_wait = 1  # Initial wait time in seconds
 
     while retry_count < max_retries:
         try:
             html_content = bookmark.get('html_content', '')
 
-            # Get the HTML soup object from the bookmark
+            # Parse HTML content
             soup = BeautifulSoup(html_content, 'html.parser')
-
-            # Extract metadata and main content
             metadata = get_page_metadata(soup)
             main_content = extract_main_content(soup)
 
-            # Prepare content for the prompt
+            # Prepare prompt
             content_parts = []
             if metadata['title']:
                 content_parts.append(f"Title: {metadata['title']}")
@@ -195,18 +198,17 @@ def generate_summary_and_assign_category(bookmark):
 
             content_text = '\n'.join(content_parts)
 
-            # Detect insufficient or erroneous content
+            # Determine prompt type
             error_keywords = ['Access Denied', 'Security Check', 'Cloudflare', 'captcha', 'unusual traffic']
             if not content_text or len(content_text.split()) < 50:
                 use_prior_knowledge = True
-                logger.info(f"Content for {bookmark.get('url')} is insufficient. Instructing LLM to use prior knowledge.")
+                logger.info(f"Content for {bookmark.get('url')} is insufficient. Using prior knowledge.")
             elif any(keyword.lower() in content_text.lower() for keyword in error_keywords):
                 use_prior_knowledge = True
-                logger.info(f"Content for {bookmark.get('url')} contains error messages. Instructing LLM to use prior knowledge.")
+                logger.info(f"Content for {bookmark.get('url')} contains error messages. Using prior knowledge.")
             else:
                 use_prior_knowledge = False
 
-            # Shortened prompts
             if use_prior_knowledge:
                 prompt = f"""
 You are a knowledgeable assistant with up-to-date information as of 2023.
@@ -235,52 +237,28 @@ Summary: [Your summary]
 Category: [One category]
 """
 
-            # Estimate tokens
-            def estimate_tokens(text):
-                return len(text) / 4  # Approximate token estimation
-
-            prompt_tokens = estimate_tokens(prompt)
-            max_tokens = 150  # Reduced from 200
-            total_tokens = prompt_tokens + max_tokens
-
-            # Calculate required delay
-            tokens_per_minute = 60000  # Adjust based on your rate limit
-            tokens_per_second = tokens_per_minute / 60
-            required_delay = total_tokens / tokens_per_second
-            sleep_time = max(required_delay, 1)
-
-            # Call the LLM via Groq Cloud API
-            response = openai.ChatCompletion.create(
-                model='llama-3.1-70b-versatile',  # Using the specified model
-                messages=[
-                    {"role": "user", "content": prompt}
-                ],
-                max_tokens=int(max_tokens),
-                temperature=0.5,
-            )
+            # Call the LLM via Groq Cloud API with rate limiting
+            with llm_rate_limiter:
+                response = openai.ChatCompletion.create(
+                    model='llama-3.1-70b-versatile',  # Ensure this is the correct model name
+                    messages=[
+                        {"role": "user", "content": prompt}
+                    ],
+                    max_tokens=150,
+                    temperature=0.5,
+                )
             content = response['choices'][0]['message']['content'].strip()
             if not content:
                 raise ValueError("Empty response received from the model.")
 
-            # Parse the response
+            # Parse response
             summary_match = re.search(r"Summary:\s*(.*)", content)
             category_match = re.search(r"Category:\s*(.*)", content)
 
-            if summary_match:
-                bookmark['summary'] = summary_match.group(1).strip()
-            else:
-                bookmark['summary'] = 'No summary available.'
-
-            if category_match:
-                category = category_match.group(1).strip().strip('"')
-                if category in CATEGORIES:
-                    bookmark['category'] = category
-                else:
-                    bookmark['category'] = 'Uncategorized'
-            else:
-                bookmark['category'] = 'Uncategorized'
+            bookmark['summary'] = summary_match.group(1).strip() if summary_match else 'No summary available.'
+            bookmark['category'] = category_match.group(1).strip().strip('"') if category_match else 'Uncategorized'
 
-            # Simple keyword-based validation (Optional)
+            # Additional validation (optional)
             summary_lower = bookmark['summary'].lower()
             url_lower = bookmark['url'].lower()
             if 'social media' in summary_lower or 'twitter' in summary_lower or 'x.com' in url_lower:
@@ -289,20 +267,23 @@ Category: [One category]
                 bookmark['category'] = 'Reference and Knowledge Bases'
 
             logger.info("Successfully generated summary and assigned category")
-            time.sleep(sleep_time)
-            break  # Exit the retry loop upon success
+            break  # Exit loop on success
 
         except openai.error.RateLimitError as e:
             retry_count += 1
-            wait_time = int(e.headers.get("Retry-After", 5))
-            logger.warning(f"Rate limit reached. Waiting for {wait_time} seconds before retrying...")
+            wait_time = base_wait * (2 ** retry_count)  # Exponential backoff
+            logger.warning(f"Rate limit reached. Waiting for {wait_time} seconds before retrying... (Attempt {retry_count}/{max_retries})")
             time.sleep(wait_time)
         except Exception as e:
             logger.error(f"Error generating summary and assigning category: {e}", exc_info=True)
-            # Ensure 'summary' is always set, even on failure
             bookmark['summary'] = 'No summary available.'
             bookmark['category'] = 'Uncategorized'
-            break  # Exit the retry loop on other exceptions
+            break  # Exit loop on non-rate limit errors
+
+    if retry_count == max_retries:
+        logger.error(f"Failed to generate summary for {bookmark.get('url')} after {max_retries} attempts.")
+        bookmark['summary'] = 'No summary available.'
+        bookmark['category'] = 'Uncategorized'
 
 def parse_bookmarks(file_content):
     """
@@ -521,12 +502,12 @@ def process_uploaded_file(file, state_bookmarks):
 
     # Fetch bookmark info concurrently
     logger.info("Fetching URL info concurrently")
-    with ThreadPoolExecutor(max_workers=10) as executor:  # Adjusted max_workers
+    with ThreadPoolExecutor(max_workers=5) as executor:  # Reduced max_workers from 10 to 5
         executor.map(fetch_url_info, bookmarks)
 
     # Generate summaries and assign categories
     logger.info("Generating summaries and assigning categories")
-    with ThreadPoolExecutor(max_workers=3) as executor:  # Adjusted max_workers
+    with ThreadPoolExecutor(max_workers=2) as executor:  # Reduced max_workers from 3 to 2
         executor.map(generate_summary_and_assign_category, bookmarks)
 
     # Log bookmarks to verify 'summary' and 'category' presence
@@ -725,15 +706,16 @@ Bookmarks:
 Provide a concise and helpful response.
 """
 
-        # Call the LLM via Groq Cloud API
-        response = openai.ChatCompletion.create(
-            model='llama-3.1-70b-versatile',  # Using the specified model
-            messages=[
-                {"role": "user", "content": prompt}
-            ],
-            max_tokens=300,
-            temperature=0.7,
-        )
+        # Call the LLM via Groq Cloud API with rate limiting
+        with llm_rate_limiter:
+            response = openai.ChatCompletion.create(
+                model='llama-3.1-70b-versatile',  # Ensure this is the correct model name
+                messages=[
+                    {"role": "user", "content": prompt}
+                ],
+                max_tokens=300,
+                temperature=0.7,
+            )
         answer = response['choices'][0]['message']['content'].strip()
         logger.info("Chatbot response generated")
         return chat_history + [{"role": "user", "content": user_query}, {"role": "assistant", "content": answer}]