# app.py import gradio as gr from bs4 import BeautifulSoup from sentence_transformers import SentenceTransformer import faiss import numpy as np import requests import time import re import logging import os import sys import threading from queue import Queue, Empty import json from concurrent.futures import ThreadPoolExecutor # Import OpenAI library import openai # Suppress only the single warning from urllib3 needed. import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # Set up logging to output to the console logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) # Create a console handler console_handler = logging.StreamHandler(sys.stdout) console_handler.setLevel(logging.INFO) # Create a formatter and set it for the handler formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s %(message)s') console_handler.setFormatter(formatter) # Add the handler to the logger logger.addHandler(console_handler) # Initialize variables and models logger.info("Initializing variables and models") embedding_model = SentenceTransformer('all-MiniLM-L6-v2') faiss_index = None bookmarks = [] fetch_cache = {} # Lock for thread-safe operations lock = threading.Lock() # Define the categories CATEGORIES = [ "Social Media", "News and Media", "Education and Learning", "Entertainment", "Shopping and E-commerce", "Finance and Banking", "Technology", "Health and Fitness", "Travel and Tourism", "Food and Recipes", "Sports", "Arts and Culture", "Government and Politics", "Business and Economy", "Science and Research", "Personal Blogs and Journals", "Job Search and Careers", "Music and Audio", "Videos and Movies", "Reference and Knowledge Bases", "Dead Link", "Uncategorized", ] # Set up Groq Cloud API key and base URL GROQ_API_KEY = os.getenv('GROQ_API_KEY') if not GROQ_API_KEY: logger.error("GROQ_API_KEY environment variable not set.") openai.api_key = GROQ_API_KEY openai.api_base = "https://api.groq.com/openai/v1" # Rate Limiter Configuration RPM_LIMIT = 60 # Requests per minute (adjust based on your API's limit) TPM_LIMIT = 60000 # Tokens per minute (adjust based on your API's limit) BATCH_SIZE = 5 # Number of bookmarks per batch # Implementing a Token Bucket Rate Limiter class TokenBucket: def __init__(self, rate, capacity): self.rate = rate # tokens per second self.capacity = capacity self.tokens = capacity self.timestamp = time.time() self.lock = threading.Lock() def consume(self, tokens=1): with self.lock: now = time.time() elapsed = now - self.timestamp refill = elapsed * self.rate self.tokens = min(self.capacity, self.tokens + refill) self.timestamp = now if self.tokens >= tokens: self.tokens -= tokens return True else: return False def wait_for_token(self, tokens=1): while not self.consume(tokens): time.sleep(0.05) # Initialize rate limiters rpm_rate = RPM_LIMIT / 60 # tokens per second tpm_rate = TPM_LIMIT / 60 # tokens per second rpm_bucket = TokenBucket(rate=rpm_rate, capacity=RPM_LIMIT) tpm_bucket = TokenBucket(rate=tpm_rate, capacity=TPM_LIMIT) # Queue for LLM tasks llm_queue = Queue() def categorize_based_on_summary(summary, url): """ Assign category based on keywords in the summary or URL. """ summary_lower = summary.lower() url_lower = url.lower() if 'social media' in summary_lower or 'twitter' in summary_lower or 'x.com' in url_lower: return 'Social Media' elif 'wikipedia' in url_lower: return 'Reference and Knowledge Bases' elif 'cloud computing' in summary_lower or 'aws' in summary_lower: return 'Technology' elif 'news' in summary_lower or 'media' in summary_lower: return 'News and Media' elif 'education' in summary_lower or 'learning' in summary_lower: return 'Education and Learning' # Add more conditions as needed else: return 'Uncategorized' def validate_category(bookmark): """ Further validate and adjust the category if needed. """ # Example: Specific cases based on URL url_lower = bookmark['url'].lower() if 'facebook' in url_lower or 'x.com' in url_lower: return 'Social Media' elif 'wikipedia' in url_lower: return 'Reference and Knowledge Bases' elif 'aws.amazon.com' in url_lower: return 'Technology' # Add more specific cases as needed else: return bookmark['category'] def extract_main_content(soup): """ Extract the main content from a webpage while filtering out boilerplate content. """ if not soup: return "" # Remove unwanted elements for element in soup(['script', 'style', 'header', 'footer', 'nav', 'aside', 'form', 'noscript']): element.decompose() # Extract text from
tags p_tags = soup.find_all('p') if p_tags: content = ' '.join([p.get_text(strip=True, separator=' ') for p in p_tags]) else: # Fallback to body content content = soup.get_text(separator=' ', strip=True) # Clean up the text content = re.sub(r'\s+', ' ', content) # Truncate content to a reasonable length (e.g., 1500 words) words = content.split() if len(words) > 1500: content = ' '.join(words[:1500]) return content def get_page_metadata(soup): """ Extract metadata from the webpage including title, description, and keywords. """ metadata = { 'title': '', 'description': '', 'keywords': '' } if not soup: return metadata # Get title title_tag = soup.find('title') if title_tag and title_tag.string: metadata['title'] = title_tag.string.strip() # Get meta description meta_desc = ( soup.find('meta', attrs={'name': 'description'}) or soup.find('meta', attrs={'property': 'og:description'}) or soup.find('meta', attrs={'name': 'twitter:description'}) ) if meta_desc: metadata['description'] = meta_desc.get('content', '').strip() # Get meta keywords meta_keywords = soup.find('meta', attrs={'name': 'keywords'}) if meta_keywords: metadata['keywords'] = meta_keywords.get('content', '').strip() # Get OG title if main title is empty if not metadata['title']: og_title = soup.find('meta', attrs={'property': 'og:title'}) if og_title: metadata['title'] = og_title.get('content', '').strip() return metadata def llm_worker(): """ Worker thread to process LLM tasks from the queue while respecting rate limits. """ logger.info("LLM worker started.") while True: batch = [] try: # Collect bookmarks up to BATCH_SIZE while len(batch) < BATCH_SIZE: bookmark = llm_queue.get(timeout=1) if bookmark is None: # Shutdown signal logger.info("LLM worker shutting down.") return if not bookmark.get('dead_link') and not bookmark.get('slow_link'): batch.append(bookmark) else: # Skip processing for dead or slow links bookmark['summary'] = 'No summary available.' bookmark['category'] = 'Uncategorized' llm_queue.task_done() except Empty: pass # No more bookmarks at the moment if batch: try: # Rate Limiting rpm_bucket.wait_for_token() # Estimate tokens: prompt + max_tokens # Here, we assume max_tokens=150 per bookmark total_tokens = 150 * len(batch) tpm_bucket.wait_for_token(tokens=total_tokens) # Prepare prompt prompt = ''' You are an assistant that creates concise webpage summaries and assigns categories. Provide summaries and categories for the following bookmarks: ''' for idx, bookmark in enumerate(batch, 1): prompt += f'Bookmark {idx}:\nURL: {bookmark["url"]}\nTitle: {bookmark["title"]}\n\n' # Corrected f-string without backslashes categories_str = ', '.join([f'"{cat}"' for cat in CATEGORIES]) prompt += f"Categories:\n{categories_str}\n\n" prompt += "Format your response as a JSON object where each key is the bookmark URL and the value is another JSON object containing 'summary' and 'category'.\n\n" prompt += "Example:\n" prompt += "{\n" prompt += ' "https://example.com": {\n' prompt += ' "summary": "This is an example summary.",\n' prompt += ' "category": "Technology"\n' prompt += " }\n" prompt += "}\n\n" prompt += "Now, provide the summaries and categories for the bookmarks listed above." response = openai.ChatCompletion.create( model='llama-3.1-70b-versatile', # Retaining the original model messages=[ {"role": "user", "content": prompt} ], max_tokens=150 * len(batch), temperature=0.5, ) content = response['choices'][0]['message']['content'].strip() if not content: raise ValueError("Empty response received from the model.") # Parse JSON response try: json_response = json.loads(content) for bookmark in batch: url = bookmark['url'] if url in json_response: summary = json_response[url].get('summary', '').strip() category = json_response[url].get('category', '').strip() if not summary: summary = 'No summary available.' bookmark['summary'] = summary if category in CATEGORIES: bookmark['category'] = category else: # Fallback to keyword-based categorization bookmark['category'] = categorize_based_on_summary(summary, url) else: logger.warning(f"No data returned for {url}. Using fallback methods.") bookmark['summary'] = 'No summary available.' bookmark['category'] = 'Uncategorized' # Additional keyword-based validation bookmark['category'] = validate_category(bookmark) logger.info(f"Processed bookmark: {url}") except json.JSONDecodeError: logger.error("Failed to parse JSON response from LLM. Using fallback methods.") for bookmark in batch: bookmark['summary'] = 'No summary available.' bookmark['category'] = categorize_based_on_summary(bookmark.get('summary', ''), bookmark['url']) bookmark['category'] = validate_category(bookmark) except Exception as e: logger.error(f"Error processing LLM response: {e}", exc_info=True) for bookmark in batch: bookmark['summary'] = 'No summary available.' bookmark['category'] = 'Uncategorized' except openai.error.RateLimitError: logger.warning(f"LLM Rate limit reached. Retrying after 60 seconds.") # Re-enqueue the entire batch for retry for bookmark in batch: llm_queue.put(bookmark) time.sleep(60) # Wait before retrying continue # Skip the rest and retry except Exception as e: logger.error(f"Error during LLM processing: {e}", exc_info=True) for bookmark in batch: bookmark['summary'] = 'No summary available.' bookmark['category'] = 'Uncategorized' finally: # Mark all bookmarks in the batch as done for _ in batch: llm_queue.task_done() def parse_bookmarks(file_content): """ Parse bookmarks from HTML file. """ logger.info("Parsing bookmarks") try: soup = BeautifulSoup(file_content, 'html.parser') extracted_bookmarks = [] for link in soup.find_all('a'): url = link.get('href') title = link.text.strip() if url and title: if url.startswith('http://') or url.startswith('https://'): extracted_bookmarks.append({'url': url, 'title': title}) else: logger.info(f"Skipping non-http/https URL: {url}") logger.info(f"Extracted {len(extracted_bookmarks)} bookmarks") return extracted_bookmarks except Exception as e: logger.error("Error parsing bookmarks: %s", e, exc_info=True) raise def fetch_url_info(bookmark): """ Fetch information about a URL. """ url = bookmark['url'] if url in fetch_cache: with lock: bookmark.update(fetch_cache[url]) return try: logger.info(f"Fetching URL info for: {url}") headers = { 'User-Agent': 'Mozilla/5.0', 'Accept-Language': 'en-US,en;q=0.9', } response = requests.get(url, headers=headers, timeout=5, verify=False, allow_redirects=True) bookmark['etag'] = response.headers.get('ETag', 'N/A') bookmark['status_code'] = response.status_code content = response.text logger.info(f"Fetched content length for {url}: {len(content)} characters") if response.status_code >= 500: bookmark['dead_link'] = True bookmark['description'] = '' bookmark['html_content'] = '' logger.warning(f"Dead link detected: {url} with status {response.status_code}") else: bookmark['dead_link'] = False bookmark['html_content'] = content bookmark['description'] = '' logger.info(f"Fetched information for {url}") except requests.exceptions.Timeout: bookmark['dead_link'] = False bookmark['etag'] = 'N/A' bookmark['status_code'] = 'Timeout' bookmark['description'] = '' bookmark['html_content'] = '' bookmark['slow_link'] = True logger.warning(f"Timeout while fetching {url}. Marking as 'Slow'.") except Exception as e: bookmark['dead_link'] = True bookmark['etag'] = 'N/A' bookmark['status_code'] = 'Error' bookmark['description'] = '' bookmark['html_content'] = '' logger.error(f"Error fetching URL info for {url}: {e}", exc_info=True) finally: with lock: fetch_cache[url] = { 'etag': bookmark.get('etag'), 'status_code': bookmark.get('status_code'), 'dead_link': bookmark.get('dead_link'), 'description': bookmark.get('description'), 'html_content': bookmark.get('html_content', ''), 'slow_link': bookmark.get('slow_link', False), } def vectorize_and_index(bookmarks_list): """ Create vector embeddings for bookmarks and build FAISS index with ID mapping. """ global faiss_index logger.info("Vectorizing summaries and building FAISS index") try: summaries = [bookmark['summary'] for bookmark in bookmarks_list] embeddings = embedding_model.encode(summaries) dimension = embeddings.shape[1] index = faiss.IndexIDMap(faiss.IndexFlatL2(dimension)) ids = np.array([bookmark['id'] for bookmark in bookmarks_list], dtype=np.int64) index.add_with_ids(np.array(embeddings).astype('float32'), ids) faiss_index = index logger.info("FAISS index built successfully with IDs") return index except Exception as e: logger.error(f"Error in vectorizing and indexing: {e}", exc_info=True) raise def display_bookmarks(): """ Generate HTML display for bookmarks. """ logger.info("Generating HTML display for bookmarks") cards = '' for i, bookmark in enumerate(bookmarks): index = i + 1 if bookmark.get('dead_link'): status = "❌ Dead Link" card_style = "border: 2px solid red;" text_style = "color: white;" summary = 'No summary available.' elif bookmark.get('slow_link'): status = "⏳ Slow Response" card_style = "border: 2px solid orange;" text_style = "color: white;" summary = bookmark.get('summary', 'No summary available.') else: status = "✅ Active" card_style = "border: 2px solid green;" text_style = "color: white;" summary = bookmark.get('summary', 'No summary available.') title = bookmark['title'] url = bookmark['url'] etag = bookmark.get('etag', 'N/A') category = bookmark.get('category', 'Uncategorized') # Escape HTML content to prevent XSS attacks from html import escape title = escape(title) url = escape(url) summary = escape(summary) category = escape(category) card_html = f'''