# app.py import gradio as gr from bs4 import BeautifulSoup from sentence_transformers import SentenceTransformer import faiss import numpy as np import asyncio import aiohttp import re import base64 import logging import os import sys # Import OpenAI library import openai # Set up logging to output to the console logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) # Create a console handler console_handler = logging.StreamHandler(sys.stdout) console_handler.setLevel(logging.INFO) # Create a formatter and set it for the handler formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s %(message)s') console_handler.setFormatter(formatter) # Add the handler to the logger logger.addHandler(console_handler) # Initialize models and variables logger.info("Initializing models and variables") embedding_model = SentenceTransformer('all-MiniLM-L6-v2') faiss_index = None bookmarks = [] fetch_cache = {} # Define the categories CATEGORIES = [ "Social Media", "News and Media", "Education and Learning", "Entertainment", "Shopping and E-commerce", "Finance and Banking", "Technology", "Health and Fitness", "Travel and Tourism", "Food and Recipes", "Sports", "Arts and Culture", "Government and Politics", "Business and Economy", "Science and Research", "Personal Blogs and Journals", "Job Search and Careers", "Music and Audio", "Videos and Movies", "Reference and Knowledge Bases", "Dead Link", "Uncategorized", ] # Set up Groq Cloud API key and base URL GROQ_API_KEY = os.getenv('GROQ_API_KEY') if not GROQ_API_KEY: logger.error("GROQ_API_KEY environment variable not set.") # Set OpenAI API key and base URL to use Groq Cloud API openai.api_key = GROQ_API_KEY openai.api_base = "https://api.groq.com/openai/v1" def extract_main_content(soup): """ Extract the main content from a webpage while filtering out boilerplate content. """ # Remove script and style elements for element in soup(['script', 'style', 'header', 'footer', 'nav', 'ads', 'sidebar']): element.decompose() # Get text from specific content tags first main_content_tags = soup.find_all(['article', 'main', 'div.content', 'div.post']) if main_content_tags: content = ' '.join([tag.get_text(strip=True, separator=' ') for tag in main_content_tags]) else: # Fallback to body content content = soup.body.get_text(strip=True, separator=' ') if soup.body else soup.get_text(strip=True, separator=' ') # Clean up the text content = ' '.join(content.split()) # Limit content length to avoid token limits return content[:3000] def get_page_metadata(soup): """ Extract metadata from the webpage including title, description, and keywords. """ metadata = { 'title': '', 'description': '', 'keywords': '' } # Get title title_tag = soup.find('title') if title_tag: metadata['title'] = title_tag.string.strip() # Get meta description meta_desc = soup.find('meta', attrs={'name': 'description'}) or \ soup.find('meta', attrs={'property': 'og:description'}) if meta_desc: metadata['description'] = meta_desc.get('content', '').strip() # Get meta keywords meta_keywords = soup.find('meta', attrs={'name': 'keywords'}) if meta_keywords: metadata['keywords'] = meta_keywords.get('content', '').strip() return metadata def generate_summary(bookmark): """ Generate a comprehensive summary for a bookmark using available content and LLM. """ logger.info(f"Generating summary for bookmark: {bookmark.get('url')}") try: # Get the HTML soup object from the bookmark if it exists soup = BeautifulSoup(bookmark.get('html_content', ''), 'html.parser') # Step 1: Try to get description from metadata metadata = get_page_metadata(soup) if metadata['description']: logger.info("Using meta description for summary") bookmark['summary'] = metadata['description'] return bookmark # Step 2: If no description, extract main content content = extract_main_content(soup) if not content: logger.warning("No content extracted from page") # Fallback to title if available if metadata['title']: bookmark['summary'] = f"Page title: {metadata['title']}" return bookmark bookmark['summary'] = bookmark.get('title', 'No summary available.') return bookmark # Step 3: Generate summary using LLM try: # Prepare context for LLM prompt = f""" Webpage Title: {metadata['title']} Keywords: {metadata['keywords']} Content: {content} Please provide a concise summary (2-3 sentences) of this webpage's main content. Focus on what the page is about and its key information. Be factual and objective. """ response = openai.ChatCompletion.create( model='llama3-8b-8192', messages=[ {"role": "system", "content": "You are a helpful assistant that creates concise webpage summaries."}, {"role": "user", "content": prompt} ], max_tokens=150, temperature=0.5, ) summary = response['choices'][0]['message']['content'].strip() logger.info("Successfully generated LLM summary") bookmark['summary'] = summary return bookmark except Exception as e: logger.error(f"Error generating LLM summary: {e}") # Fallback to extracted content bookmark['summary'] = ' '.join(content.split()[:50]) + '...' return bookmark except Exception as e: logger.error(f"Error in generate_summary: {e}") # Final fallback bookmark['summary'] = bookmark.get('title', 'No summary available.') return bookmark # Function to parse bookmarks from HTML def parse_bookmarks(file_content): logger.info("Parsing bookmarks") try: soup = BeautifulSoup(file_content, 'html.parser') extracted_bookmarks = [] for link in soup.find_all('a'): url = link.get('href') title = link.text.strip() if url and title: extracted_bookmarks.append({'url': url, 'title': title}) logger.info(f"Extracted {len(extracted_bookmarks)} bookmarks") return extracted_bookmarks except Exception as e: logger.error("Error parsing bookmarks: %s", e) raise # Asynchronous function to fetch URL info async def fetch_url_info(session, bookmark): url = bookmark['url'] if url in fetch_cache: bookmark.update(fetch_cache[url]) return bookmark try: logger.info(f"Fetching URL info for: {url}") async with session.get(url, timeout=10) as response: bookmark['etag'] = response.headers.get('ETag', 'N/A') bookmark['status_code'] = response.status if response.status >= 400: bookmark['dead_link'] = True bookmark['description'] = '' logger.warning(f"Dead link detected: {url} with status {response.status}") else: bookmark['dead_link'] = False content = await response.text() bookmark['html_content'] = content # Store HTML content for summary generation soup = BeautifulSoup(content, 'html.parser') bookmark['description'] = '' # Will be set by generate_summary function logger.info(f"Fetched information for {url}") except Exception as e: bookmark['dead_link'] = True bookmark['etag'] = 'N/A' bookmark['status_code'] = 'N/A' bookmark['description'] = '' bookmark['html_content'] = '' logger.error(f"Error fetching URL info for {url}: {e}") finally: fetch_cache[url] = { 'etag': bookmark.get('etag'), 'status_code': bookmark.get('status_code'), 'dead_link': bookmark.get('dead_link'), 'description': bookmark.get('description'), 'html_content': bookmark.get('html_content', '') } return bookmark # Asynchronous processing of bookmarks async def process_bookmarks_async(bookmarks_list): logger.info("Processing bookmarks asynchronously") try: async with aiohttp.ClientSession() as session: tasks = [] for bookmark in bookmarks_list: task = asyncio.ensure_future(fetch_url_info(session, bookmark)) tasks.append(task) await asyncio.gather(*tasks) logger.info("Completed processing bookmarks asynchronously") except Exception as e: logger.error(f"Error in asynchronous processing of bookmarks: {e}") raise # Assign category to a bookmark def assign_category(bookmark): if bookmark.get('dead_link'): bookmark['category'] = 'Dead Link' logger.info(f"Assigned category 'Dead Link' to bookmark: {bookmark.get('url')}") return bookmark summary = bookmark.get('summary', '').lower() assigned_category = 'Uncategorized' # Keywords associated with each category category_keywords = { "Social Media": ["social media", "networking", "friends", "connect", "posts", "profile"], "News and Media": ["news", "journalism", "media", "headlines", "breaking news"], "Education and Learning": ["education", "learning", "courses", "tutorial", "university", "academy", "study"], "Entertainment": ["entertainment", "movies", "tv shows", "games", "comics", "fun"], "Shopping and E-commerce": ["shopping", "e-commerce", "buy", "sell", "marketplace", "deals", "store"], "Finance and Banking": ["finance", "banking", "investment", "money", "economy", "stock", "trading"], "Technology": ["technology", "tech", "gadgets", "software", "computers", "innovation"], "Health and Fitness": ["health", "fitness", "medical", "wellness", "exercise", "diet"], "Travel and Tourism": ["travel", "tourism", "destinations", "hotels", "flights", "vacation"], "Food and Recipes": ["food", "recipes", "cooking", "cuisine", "restaurant", "dining"], "Sports": ["sports", "scores", "teams", "athletics", "matches", "leagues"], "Arts and Culture": ["arts", "culture", "museum", "gallery", "exhibition", "artistic"], "Government and Politics": ["government", "politics", "policy", "election", "public service"], "Business and Economy": ["business", "corporate", "industry", "economy", "markets"], "Science and Research": ["science", "research", "experiment", "laboratory", "study", "scientific"], "Personal Blogs and Journals": ["blog", "journal", "personal", "diary", "thoughts", "opinions"], "Job Search and Careers": ["jobs", "careers", "recruitment", "resume", "employment", "hiring"], "Music and Audio": ["music", "audio", "songs", "albums", "artists", "bands"], "Videos and Movies": ["video", "movies", "film", "clips", "trailers", "cinema"], "Reference and Knowledge Bases": ["reference", "encyclopedia", "dictionary", "wiki", "knowledge", "information"], } for category, keywords in category_keywords.items(): for keyword in keywords: if re.search(r'\b' + re.escape(keyword) + r'\b', summary): assigned_category = category logger.info(f"Assigned category '{assigned_category}' to bookmark: {bookmark.get('url')}") break if assigned_category != 'Uncategorized': break bookmark['category'] = assigned_category if assigned_category == 'Uncategorized': logger.info(f"No matching category found for bookmark: {bookmark.get('url')}") return bookmark # Vectorize summaries and build FAISS index def vectorize_and_index(bookmarks_list): logger.info("Vectorizing summaries and building FAISS index") try: summaries = [bookmark['summary'] for bookmark in bookmarks_list] embeddings = embedding_model.encode(summaries) dimension = embeddings.shape[1] faiss_idx = faiss.IndexFlatL2(dimension) faiss_idx.add(np.array(embeddings)) logger.info("FAISS index built successfully") return faiss_idx, embeddings except Exception as e: logger.error(f"Error in vectorizing and indexing: {e}") raise # Generate HTML display for bookmarks def display_bookmarks(): logger.info("Generating HTML display for bookmarks") cards = '' for i, bookmark in enumerate(bookmarks): index = i + 1 # Start index at 1 status = "❌ Dead Link" if bookmark.get('dead_link') else "✅ Active" title = bookmark['title'] url = bookmark['url'] etag = bookmark.get('etag', 'N/A') summary = bookmark.get('summary', '') category = bookmark.get('category', 'Uncategorized') # Apply inline styles using CSS variables if bookmark.get('dead_link'): card_style = "border: 2px solid var(--error-color);" text_style = "color: var(--error-color);" else: card_style = "border: 2px solid var(--success-color);" text_style = "color: var(--text-color);" card_html = f'''

{index}. {title} {status}

Category: {category}

URL: {url}

ETag: {etag}

Summary: {summary}

''' cards += card_html logger.info("HTML display generated") return cards # Process the uploaded file def process_uploaded_file(file): global bookmarks, faiss_index logger.info("Processing uploaded file") if file is None: logger.warning("No file uploaded") return "Please upload a bookmarks HTML file.", '', gr.update(choices=[]), display_bookmarks() try: file_content = file.decode('utf-8') except UnicodeDecodeError as e: logger.error(f"Error decoding the file: {e}") return "Error decoding the file. Please ensure it's a valid HTML file.", '', gr.update(choices=[]), display_bookmarks() try: bookmarks = parse_bookmarks(file_content) except Exception as e: logger.error(f"Error parsing bookmarks: {e}") return "Error parsing the bookmarks HTML file.", '', gr.update(choices=[]), display_bookmarks() if not bookmarks: logger.warning("No bookmarks found in the uploaded file") return "No bookmarks found in the uploaded file.", '', gr.update(choices=[]), display_bookmarks() # Asynchronously fetch bookmark info try: asyncio.run(process_bookmarks_async(bookmarks)) except Exception as e: logger.error(f"Error processing bookmarks asynchronously: {e}") return "Error processing bookmarks.", '', gr.update(choices=[]), display_bookmarks() # Generate summaries and assign categories for bookmark in bookmarks: generate_summary(bookmark) assign_category(bookmark) try: faiss_index, embeddings = vectorize_and_index(bookmarks) except Exception as e: logger.error(f"Error building FAISS index: {e}") return "Error building search index.", '', gr.update(choices=[]), display_bookmarks() message = f"✅ Successfully processed {len(bookmarks)} bookmarks." logger.info(message) bookmark_html = display_bookmarks() # Update bookmark_selector choices choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" for i, bookmark in enumerate(bookmarks)] bookmark_selector_update = gr.update(choices=choices, value=[]) # Update bookmark_display_manage bookmark_display_manage_update = display_bookmarks() return message, bookmark_html, bookmark_selector_update, bookmark_display_manage_update # Delete selected bookmarks def delete_selected_bookmarks(selected_indices): global bookmarks, faiss_index if not selected_indices: return "⚠️ No bookmarks selected.", gr.update(choices=[]), display_bookmarks() indices = [int(s.split('.')[0])-1 for s in selected_indices] indices = sorted(indices, reverse=True) for idx in indices: if 0 <= idx < len(bookmarks): logger.info(f"Deleting bookmark at index {idx + 1}") bookmarks.pop(idx) if bookmarks: faiss_index, embeddings = vectorize_and_index(bookmarks) else: faiss_index = None message = "🗑️ Selected bookmarks deleted successfully." logger.info(message) # Update bookmark_selector choices choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" for i, bookmark in enumerate(bookmarks)] bookmark_selector_update = gr.update(choices=choices, value=[]) # Update bookmarks display bookmarks_html = display_bookmarks() return message, bookmark_selector_update, bookmarks_html # Edit category of selected bookmarks def edit_selected_bookmarks_category(selected_indices, new_category): if not selected_indices: return "⚠️ No bookmarks selected.", '', gr.update() if not new_category: return "⚠️ No new category selected.", '', gr.update() indices = [int(s.split('.')[0])-1 for s in selected_indices] for idx in indices: if 0 <= idx < len(bookmarks): bookmarks[idx]['category'] = new_category logger.info(f"Updated category for bookmark {idx + 1} to {new_category}") message = "✏️ Category updated for selected bookmarks." logger.info(message) # Update bookmark_selector choices choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" for i, bookmark in enumerate(bookmarks)] bookmark_selector_update = gr.update(choices=choices, value=[]) # Update bookmarks display bookmarks_html = display_bookmarks() return message, bookmark_selector_update, bookmarks_html # Export bookmarks to HTML def export_bookmarks(): if not bookmarks: logger.warning("No bookmarks to export") return "⚠️ No bookmarks to export." try: logger.info("Exporting bookmarks to HTML") # Create an HTML content similar to the imported bookmarks file soup = BeautifulSoup("Bookmarks

Bookmarks

", 'html.parser') dl = soup.new_tag('DL') for bookmark in bookmarks: dt = soup.new_tag('DT') a = soup.new_tag('A', href=bookmark['url']) a.string = bookmark['title'] dt.append(a) dl.append(dt) soup.append(dl) html_content = str(soup) # Encode the HTML content to base64 for download b64 = base64.b64encode(html_content.encode()).decode() href = f'data:text/html;base64,{b64}' logger.info("Bookmarks exported successfully") return f'💾 Download Exported Bookmarks' except Exception as e: logger.error(f"Error exporting bookmarks: {e}") return "⚠️ Error exporting bookmarks." # Chatbot response using Groq Cloud API def chatbot_response(user_query): if not GROQ_API_KEY: logger.warning("GROQ_API_KEY not set.") return "⚠️ API key not set. Please set the GROQ_API_KEY environment variable in the Hugging Face Space settings." if not bookmarks: logger.warning("No bookmarks available for chatbot") return "⚠️ No bookmarks available. Please upload and process your bookmarks first." logger.info(f"Chatbot received query: {user_query}") try: # Limit the number of bookmarks to prevent exceeding token limits max_bookmarks = 50 # Adjust as needed bookmark_data = "" for idx, bookmark in enumerate(bookmarks[:max_bookmarks]): bookmark_data += f"{idx+1}. Title: {bookmark['title']}\nURL: {bookmark['url']}\nSummary: {bookmark['summary']}\n\n" # Construct the prompt prompt = f""" You are an assistant that helps users find relevant bookmarks from their collection based on their queries. User Query: {user_query} Bookmarks: {bookmark_data} Please identify the most relevant bookmarks that match the user's query. Provide a concise list including the index, title, URL, and a brief summary. """ # Call the Groq Cloud API via the OpenAI client response = openai.ChatCompletion.create( model='llama3-8b-8192', messages=[ {"role": "system", "content": "You help users find relevant bookmarks based on their queries."}, {"role": "user", "content": prompt} ], max_tokens=500, temperature=0.7, ) # Extract the response text answer = response['choices'][0]['message']['content'].strip() logger.info("Chatbot response generated using Groq Cloud API") return answer except Exception as e: error_message = f"⚠️ Error processing your query: {str(e)}" logger.error(error_message) print(error_message) # Ensure error appears in Hugging Face Spaces logs return error_message # Build the Gradio app def build_app(): try: logger.info("Building Gradio app") with gr.Blocks(css="app.css") as demo: # General Overview gr.Markdown(""" # 📚 SmartMarks - AI Browser Bookmarks Manager Welcome to **SmartMarks**, your intelligent assistant for managing browser bookmarks. SmartMarks leverages AI to help you organize, search, and interact with your bookmarks seamlessly. Whether you're looking to categorize your links, retrieve information quickly, or maintain an updated list, SmartMarks has you covered. --- ## 🚀 **How to Use SmartMarks** SmartMarks is divided into three main sections: 1. **📂 Upload and Process Bookmarks:** Import your existing bookmarks and let SmartMarks analyze and categorize them for you. 2. **💬 Chat with Bookmarks:** Interact with your bookmarks using natural language queries to find relevant links effortlessly. 3. **🛠️ Manage Bookmarks:** View, edit, delete, and export your bookmarks with ease. Navigate through the tabs to explore each feature in detail. """) # Upload and Process Bookmarks Tab with gr.Tab("Upload and Process Bookmarks"): gr.Markdown(""" ## 📂 **Upload and Process Bookmarks** ### 📝 **Steps to Upload and Process:** 1. **🔽 Upload Bookmarks File:** - Click on the **"Upload Bookmarks HTML File"** button. - Select your browser's exported bookmarks HTML file from your device. 2. **⚙️ Process Bookmarks:** - After uploading, click on the **"Process Bookmarks"** button. - SmartMarks will parse your bookmarks, fetch additional information, generate summaries, and categorize each link based on predefined categories. 3. **📄 View Processed Bookmarks:** - Once processing is complete, your bookmarks will be displayed in an organized and visually appealing format below. """) upload = gr.File(label="📁 Upload Bookmarks HTML File", type='binary') process_button = gr.Button("⚙️ Process Bookmarks") output_text = gr.Textbox(label="✅ Output", interactive=False) bookmark_display = gr.HTML(label="📄 Bookmarks") # Initialize Manage Bookmarks components bookmark_selector = gr.CheckboxGroup(label="✅ Select Bookmarks", choices=[]) bookmark_display_manage = gr.HTML(label="📄 Manage Bookmarks Display") process_button.click( process_uploaded_file, inputs=upload, outputs=[output_text, bookmark_display, bookmark_selector, bookmark_display_manage] ) # Chat with Bookmarks Tab with gr.Tab("Chat with Bookmarks"): gr.Markdown(""" ## 💬 **Chat with Bookmarks** ### 🤖 **How to Interact:** 1. **✍️ Enter Your Query:** - In the **"Ask about your bookmarks"** textbox, type your question or keyword related to your bookmarks. For example, "Do I have any bookmarks about GenerativeAI?" 2. **📨 Submit Your Query:** - Click the **"Send"** button to submit your query. 3. **📈 Receive AI-Driven Responses:** - SmartMarks will analyze your query and provide relevant bookmarks that match your request, making it easier to find specific links without manual searching. """) user_input = gr.Textbox(label="✍️ Ask about your bookmarks", placeholder="e.g., Do I have any bookmarks about GenerativeAI?") chat_output = gr.Textbox(label="💬 Chatbot Response", interactive=False) chat_button = gr.Button("📨 Send") chat_button.click( chatbot_response, inputs=user_input, outputs=chat_output ) # Manage Bookmarks Tab with gr.Tab("Manage Bookmarks"): gr.Markdown(""" ## 🛠️ **Manage Bookmarks** ### 🗂️ **Features:** 1. **👁️ View Bookmarks:** - All your processed bookmarks are displayed here with their respective categories and summaries. 2. **✅ Select Bookmarks:** - Use the checkboxes next to each bookmark to select one, multiple, or all bookmarks you wish to manage. 3. **🗑️ Delete Selected Bookmarks:** - After selecting the desired bookmarks, click the **"Delete Selected Bookmarks"** button to remove them from your list. 4. **✏️ Edit Categories:** - Select the bookmarks you want to re-categorize. - Choose a new category from the dropdown menu labeled **"New Category"**. - Click the **"Edit Category of Selected Bookmarks"** button to update their categories. 5. **💾 Export Bookmarks:** - Click the **"Export Bookmarks"** button to download your updated bookmarks as an HTML file. - This file can be uploaded back to your browser to reflect the changes made within SmartMarks. """) manage_output = gr.Textbox(label="🔄 Manage Output", interactive=False) bookmark_display_manage = gr.HTML(label="📄 Manage Bookmarks Display") bookmark_selector = gr.CheckboxGroup(label="✅ Select Bookmarks", choices=[]) new_category_input = gr.Dropdown(label="🆕 New Category", choices=CATEGORIES, value="Uncategorized") with gr.Row(): delete_button = gr.Button("🗑️ Delete Selected Bookmarks") edit_category_button = gr.Button("✏️ Edit Category of Selected Bookmarks") export_button = gr.Button("💾 Export Bookmarks") download_link = gr.HTML(label="📥 Download Exported Bookmarks") # Define button actions delete_button.click( delete_selected_bookmarks, inputs=bookmark_selector, outputs=[manage_output, bookmark_selector, bookmark_display_manage] ) edit_category_button.click( edit_selected_bookmarks_category, inputs=[bookmark_selector, new_category_input], outputs=[manage_output, bookmark_selector, bookmark_display_manage] ) export_button.click( export_bookmarks, inputs=None, outputs=download_link ) # Initialize display after processing bookmarks process_button.click( process_uploaded_file, inputs=upload, outputs=[output_text, bookmark_display, bookmark_selector, bookmark_display_manage] ) logger.info("Launching Gradio app") demo.launch(debug=True) except Exception as e: logger.error(f"Error building the app: {e}") print(f"Error building the app: {e}") if __name__ == "__main__": build_app()