# app.py import gradio as gr from bs4 import BeautifulSoup from sentence_transformers import SentenceTransformer import faiss import numpy as np import asyncio import aiohttp import re import base64 import logging import os import sys import uuid # For unique IDs # Import OpenAI library import openai # Set up logging to output to the console logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) # Create a console handler console_handler = logging.StreamHandler(sys.stdout) console_handler.setLevel(logging.INFO) # Create a formatter and set it for the handler formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s %(message)s') console_handler.setFormatter(formatter) # Add the handler to the logger logger.addHandler(console_handler) # Initialize models and variables logger.info("Initializing models and variables") embedding_model = SentenceTransformer('all-MiniLM-L6-v2') faiss_index = faiss.IndexIDMap(faiss.IndexFlatL2(embedding_model.get_sentence_embedding_dimension())) bookmarks = [] fetch_cache = {} # Define the categories CATEGORIES = [ "Social Media", "News and Media", "Education and Learning", "Entertainment", "Shopping and E-commerce", "Finance and Banking", "Technology", "Health and Fitness", "Travel and Tourism", "Food and Recipes", "Sports", "Arts and Culture", "Government and Politics", "Business and Economy", "Science and Research", "Personal Blogs and Journals", "Job Search and Careers", "Music and Audio", "Videos and Movies", "Reference and Knowledge Bases", "Dead Link", "Uncategorized", ] # Set up Groq Cloud API key and base URL GROQ_API_KEY = os.getenv('GROQ_API_KEY') if not GROQ_API_KEY: logger.error("GROQ_API_KEY environment variable not set.") # Set OpenAI API key and base URL to use Groq Cloud API openai.api_key = GROQ_API_KEY openai.api_base = "https://api.groq.com/openai/v1" # Corrected API base URL # Function to parse bookmarks from HTML def parse_bookmarks(file_content): logger.info("Parsing bookmarks") try: soup = BeautifulSoup(file_content, 'html.parser') extracted_bookmarks = [] for link in soup.find_all('a'): url = link.get('href') title = link.text.strip() if url and title: extracted_bookmarks.append({'url': url, 'title': title}) logger.info(f"Extracted {len(extracted_bookmarks)} bookmarks") return extracted_bookmarks except Exception as e: logger.error("Error parsing bookmarks: %s", e) raise # Asynchronous function to fetch URL info async def fetch_url_info(session, bookmark): url = bookmark['url'] if url in fetch_cache: bookmark.update(fetch_cache[url]) return bookmark try: logger.info(f"Fetching URL info for: {url}") async with session.get(url, timeout=10) as response: bookmark['etag'] = response.headers.get('ETag', 'N/A') bookmark['status_code'] = response.status if response.status >= 400: bookmark['dead_link'] = True bookmark['description'] = '' logger.warning(f"Dead link detected: {url} with status {response.status}") else: bookmark['dead_link'] = False content = await response.text() soup = BeautifulSoup(content, 'html.parser') # Extract meta description or Open Graph description meta_description = soup.find('meta', attrs={'name': 'description'}) og_description = soup.find('meta', attrs={'property': 'og:description'}) if og_description and og_description.get('content'): description = og_description.get('content') elif meta_description and meta_description.get('content'): description = meta_description.get('content') else: # If no description, extract visible text texts = soup.stripped_strings description = ' '.join(texts[:200]) # Limit to first 200 words # Generate summary using LLM description = generate_summary_with_llm(description) bookmark['description'] = description logger.info(f"Fetched description for {url}") except Exception as e: bookmark['dead_link'] = True bookmark['etag'] = 'N/A' bookmark['status_code'] = 'N/A' bookmark['description'] = '' logger.error(f"Error fetching URL info for {url}: {e}") finally: fetch_cache[url] = { 'etag': bookmark.get('etag'), 'status_code': bookmark.get('status_code'), 'dead_link': bookmark.get('dead_link'), 'description': bookmark.get('description'), } return bookmark # Asynchronous processing of bookmarks async def process_bookmarks_async(bookmarks_list): logger.info("Processing bookmarks asynchronously") try: async with aiohttp.ClientSession() as session: tasks = [] for bookmark in bookmarks_list: task = asyncio.ensure_future(fetch_url_info(session, bookmark)) tasks.append(task) await asyncio.gather(*tasks) logger.info("Completed processing bookmarks asynchronously") except Exception as e: logger.error(f"Error in asynchronous processing of bookmarks: {e}") raise # Generate summary for a bookmark using LLM def generate_summary_with_llm(text): logger.info("Generating summary with LLM") try: prompt = f"Summarize the following content in a concise manner:\n\n{text}" response = openai.ChatCompletion.create( model='gpt-3.5-turbo', # Use appropriate model messages=[ {"role": "system", "content": "You are a helpful assistant that summarizes text."}, {"role": "user", "content": prompt} ], max_tokens=150, temperature=0.5, ) summary = response['choices'][0]['message']['content'].strip() logger.info("Summary generated successfully") return summary except Exception as e: logger.error(f"Error generating summary with LLM: {e}") return "No summary available." # Generate summary for a bookmark def generate_summary(bookmark): description = bookmark.get('description', '') if description: bookmark['summary'] = description else: # Fallback summary generation title = bookmark.get('title', '') if title: bookmark['summary'] = title else: bookmark['summary'] = 'No summary available.' logger.info(f"Generated summary for bookmark: {bookmark.get('url')}") return bookmark # Assign category to a bookmark using LLM def assign_category_with_llm(summary): logger.info("Assigning category with LLM") try: categories_str = ', '.join(CATEGORIES) prompt = f"Assign the most appropriate category to the following summary from the list of categories.\n\nSummary: {summary}\n\nCategories: {categories_str}\n\nCategory:" response = openai.ChatCompletion.create( model='gpt-3.5-turbo', # Use appropriate model messages=[ {"role": "system", "content": "You are a helpful assistant that assigns categories."}, {"role": "user", "content": prompt} ], max_tokens=10, temperature=0.3, ) category = response['choices'][0]['message']['content'].strip() # Ensure the category is valid if category in CATEGORIES: logger.info(f"Assigned category '{category}' successfully") return category else: logger.warning(f"Received invalid category '{category}' from LLM. Defaulting to 'Uncategorized'.") return "Uncategorized" except Exception as e: logger.error(f"Error assigning category with LLM: {e}") return "Uncategorized" # Assign category to a bookmark def assign_category(bookmark): if bookmark.get('dead_link'): bookmark['category'] = 'Dead Link' logger.info(f"Assigned category 'Dead Link' to bookmark: {bookmark.get('url')}") return bookmark summary = bookmark.get('summary', '') if summary: category = assign_category_with_llm(summary) bookmark['category'] = category else: bookmark['category'] = 'Uncategorized' logger.info(f"No summary available to assign category for bookmark: {bookmark.get('url')}") return bookmark # Vectorize summaries and build FAISS index def vectorize_and_index(bookmarks_list): global faiss_index logger.info("Vectorizing summaries and updating FAISS index") try: summaries = [bookmark['summary'] for bookmark in bookmarks_list] embeddings = embedding_model.encode(summaries).astype('float32') ids = [uuid.uuid4().int & (1<<64)-1 for _ in bookmarks_list] # Generate unique 64-bit integer IDs faiss_index.add_with_ids(embeddings, np.array(ids)) logger.info("FAISS index updated successfully") return embeddings, ids except Exception as e: logger.error(f"Error in vectorizing and indexing: {e}") raise # Remove vectors from FAISS index by IDs def remove_from_faiss(ids_to_remove): global faiss_index logger.info(f"Removing {len(ids_to_remove)} vectors from FAISS index") try: faiss_index.remove_ids(np.array(ids_to_remove)) logger.info("Vectors removed from FAISS index successfully") except Exception as e: logger.error(f"Error removing vectors from FAISS index: {e}") # Generate HTML display for bookmarks def display_bookmarks(): logger.info("Generating HTML display for bookmarks") cards = '' for i, bookmark in enumerate(bookmarks): index = i + 1 # Start index at 1 status = "❌ Dead Link" if bookmark.get('dead_link') else "✅ Active" title = bookmark['title'] url = bookmark['url'] etag = bookmark.get('etag', 'N/A') summary = bookmark.get('summary', '') category = bookmark.get('category', 'Uncategorized') # Apply inline styles using CSS variables if bookmark.get('dead_link'): card_style = "border: 2px solid var(--error-color);" text_style = "color: var(--error-color);" else: card_style = "border: 2px solid var(--success-color);" text_style = "color: var(--text-color);" card_html = f'''

{index}. {title} {status}

Category: {category}

URL: {url}

ETag: {etag}

Summary: {summary}

''' cards += card_html logger.info("HTML display generated") return cards # Process the uploaded file def process_uploaded_file(file, state_bookmarks): global bookmarks, faiss_index logger.info("Processing uploaded file") if file is None: logger.warning("No file uploaded") return ( "⚠️ Please upload a bookmarks HTML file.", "", gr.update(choices=[]), display_bookmarks(), state_bookmarks # Return the unchanged state ) try: file_content = file.decode('utf-8') except UnicodeDecodeError as e: logger.error(f"Error decoding the file: {e}") return ( "⚠️ Error decoding the file. Please ensure it's a valid HTML file.", "", gr.update(choices=[]), display_bookmarks(), state_bookmarks # Return the unchanged state ) try: bookmarks = parse_bookmarks(file_content) except Exception as e: logger.error(f"Error parsing bookmarks: {e}") return ( "⚠️ Error parsing the bookmarks HTML file.", "", gr.update(choices=[]), display_bookmarks(), state_bookmarks # Return the unchanged state ) if not bookmarks: logger.warning("No bookmarks found in the uploaded file") return ( "⚠️ No bookmarks found in the uploaded file.", "", gr.update(choices=[]), display_bookmarks(), state_bookmarks # Return the unchanged state ) # Asynchronously fetch bookmark info try: asyncio.run(process_bookmarks_async(bookmarks)) except Exception as e: logger.error(f"Error processing bookmarks asynchronously: {e}") return ( "⚠️ Error processing bookmarks.", "", gr.update(choices=[]), display_bookmarks(), state_bookmarks # Return the unchanged state ) # Generate summaries and assign categories for bookmark in bookmarks: generate_summary(bookmark) assign_category(bookmark) try: embeddings, ids = vectorize_and_index(bookmarks) except Exception as e: logger.error(f"Error building FAISS index: {e}") return ( "⚠️ Error building search index.", "", gr.update(choices=[]), display_bookmarks(), state_bookmarks # Return the unchanged state ) # Assign unique IDs to bookmarks for bookmark, id_ in zip(bookmarks, ids): bookmark['id'] = id_ message = f"✅ Successfully processed {len(bookmarks)} bookmarks." logger.info(message) bookmark_html = display_bookmarks() # Prepare Manage Bookmarks tab outputs choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" for i, bookmark in enumerate(bookmarks)] bookmarks_html_manage = display_bookmarks() # Update the shared state updated_state = bookmarks.copy() return ( message, bookmark_html, gr.update(choices=choices, value=[]), bookmarks_html_manage, updated_state # Return the updated state ) # Delete selected bookmarks def delete_selected_bookmarks(selected_indices, state_bookmarks): if not selected_indices: return "⚠️ No bookmarks selected.", gr.update(choices=[]), display_bookmarks(), state_bookmarks bookmarks = state_bookmarks.copy() ids_to_remove = [] indices = [] for s in selected_indices: try: idx = int(s.split('.')[0]) - 1 if 0 <= idx < len(bookmarks): ids_to_remove.append(bookmarks[idx]['id']) indices.append(idx) else: logger.warning(f"Index out of range: {idx + 1}") except ValueError: logger.error(f"Invalid selection format: {s}") # Remove bookmarks from the list indices = sorted(indices, reverse=True) for idx in indices: logger.info(f"Deleting bookmark at index {idx + 1}") bookmarks.pop(idx) # Remove embeddings from FAISS index remove_from_faiss(ids_to_remove) message = "🗑️ Selected bookmarks deleted successfully." logger.info(message) # Regenerate HTML display bookmarks_html = display_bookmarks() # Update choices for selection choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" for i, bookmark in enumerate(bookmarks)] # Update the shared state updated_state = bookmarks.copy() return message, gr.update(choices=choices, value=[]), bookmarks_html, updated_state # Edit category of selected bookmarks def edit_selected_bookmarks_category(selected_indices, new_category, state_bookmarks): if not selected_indices: return ( "⚠️ No bookmarks selected.", gr.update(choices=[f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" for i, bookmark in enumerate(state_bookmarks)], value=[]), display_bookmarks(), state_bookmarks ) if not new_category: return ( "⚠️ No new category selected.", gr.update(choices=[f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" for i, bookmark in enumerate(state_bookmarks)], value=[]), display_bookmarks(), state_bookmarks ) bookmarks = state_bookmarks.copy() for s in selected_indices: try: idx = int(s.split('.')[0]) - 1 if 0 <= idx < len(bookmarks): bookmarks[idx]['category'] = new_category logger.info(f"Updated category for bookmark {idx + 1} to {new_category}") except ValueError: logger.error(f"Invalid selection format: {s}") message = "✏️ Category updated for selected bookmarks." logger.info(message) # Regenerate HTML display bookmarks_html = display_bookmarks() # Update choices for selection choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" for i, bookmark in enumerate(bookmarks)] # Update the shared state updated_state = bookmarks.copy() return message, gr.update(choices=choices, value=[]), bookmarks_html, updated_state # Export bookmarks to HTML def export_bookmarks(state_bookmarks): bookmarks = state_bookmarks if not bookmarks: logger.warning("No bookmarks to export") return "⚠️ No bookmarks to export." try: logger.info("Exporting bookmarks to HTML") # Create an HTML content similar to the imported bookmarks file soup = BeautifulSoup("Bookmarks

Bookmarks

", 'html.parser') dl = soup.new_tag('DL') for bookmark in bookmarks: dt = soup.new_tag('DT') a = soup.new_tag('A', href=bookmark['url']) a.string = bookmark['title'] dt.append(a) dl.append(dt) soup.append(dl) html_content = str(soup) # Encode the HTML content to base64 for download b64 = base64.b64encode(html_content.encode()).decode() href = f'data:text/html;base64,{b64}' logger.info("Bookmarks exported successfully") return f'💾 Download Exported Bookmarks' except Exception as e: logger.error(f"Error exporting bookmarks: {e}") return "⚠️ Error exporting bookmarks." # Chatbot response using Groq Cloud API with FAISS search def chatbot_response(user_query, state_bookmarks): if not GROQ_API_KEY: logger.warning("GROQ_API_KEY not set.") return "⚠️ API key not set. Please set the GROQ_API_KEY environment variable in the Hugging Face Space settings." bookmarks = state_bookmarks if not bookmarks: logger.warning("No bookmarks available for chatbot") return "⚠️ No bookmarks available. Please upload and process your bookmarks first." logger.info(f"Chatbot received query: {user_query}") # Prepare the prompt for the LLM using FAISS search try: # Encode the user query query_embedding = embedding_model.encode([user_query]).astype('float32') # Perform FAISS search k = 5 # Number of top results to retrieve distances, indices = faiss_index.search(query_embedding, k) # Fetch the corresponding bookmarks relevant_bookmarks = [] for idx in indices[0]: if idx == -1: continue # No more results # Find the bookmark with matching ID for bookmark in bookmarks: if bookmark.get('id') == idx: relevant_bookmarks.append(bookmark) break if not relevant_bookmarks: return "🔍 No relevant bookmarks found for your query." # Prepare the summary of relevant bookmarks bookmark_data = "" for bm in relevant_bookmarks: bookmark_data += f"Title: {bm['title']}\nURL: {bm['url']}\nSummary: {bm['summary']}\n\n" # Construct the prompt prompt = f""" You are an assistant that helps users find relevant bookmarks from their collection based on their queries. User Query: {user_query} Relevant Bookmarks: {bookmark_data} Please provide a concise summary of the most relevant bookmarks that match the user's query. """ # Call the Groq Cloud API via the OpenAI client response = openai.ChatCompletion.create( model='gpt-3.5-turbo', # Use appropriate model messages=[ {"role": "system", "content": "You help users find relevant bookmarks based on their queries."}, {"role": "user", "content": prompt} ], max_tokens=500, temperature=0.7, ) # Extract the response text answer = response['choices'][0]['message']['content'].strip() logger.info("Chatbot response generated using Groq Cloud API") return answer except Exception as e: error_message = f"⚠️ Error processing your query: {str(e)}" logger.error(error_message) print(error_message) # Ensure error appears in Hugging Face Spaces logs return error_message # Build the Gradio app def build_app(): try: logger.info("Building Gradio app") with gr.Blocks(css=""" .card { box-shadow: 0 4px 8px 0 rgba(0,0,0,0.2); transition: 0.3s; } .card:hover { box-shadow: 0 8px 16px 0 rgba(0,0,0,0.2); } /* Dynamic Theme Styling */ @media (prefers-color-scheme: dark) { body { color: white; background-color: #121212; } .card { background-color: #1e1e1e; } a { color: #bb86fc; } h1, h2, h3, p, strong { color: inherit; } } @media (prefers-color-scheme: light) { body { color: black; background-color: white; } .card { background-color: #fff; } a { color: #1a0dab; } h1, h2, h3, p, strong { color: inherit; } } """) as demo: # Shared states state_bookmarks = gr.State([]) chat_history = gr.State([]) # General Overview gr.Markdown(""" # 📚 SmartMarks - AI Browser Bookmarks Manager Welcome to **SmartMarks**, your intelligent assistant for managing browser bookmarks. SmartMarks leverages AI to help you organize, search, and interact with your bookmarks seamlessly. Whether you're looking to categorize your links, retrieve information quickly, or maintain an updated list, SmartMarks has you covered. --- ## 🚀 **How to Use SmartMarks** SmartMarks is divided into three main sections: 1. **📂 Upload and Process Bookmarks:** Import your existing bookmarks and let SmartMarks analyze and categorize them for you. 2. **💬 Chat with Bookmarks:** Interact with your bookmarks using natural language queries to find relevant links effortlessly. 3. **🛠️ Manage Bookmarks:** View, edit, delete, and export your bookmarks with ease. Navigate through the tabs to explore each feature in detail. """) # Define Manage Bookmarks components outside the tab for global access bookmark_selector = gr.CheckboxGroup(label="✅ Select Bookmarks", choices=[]) bookmark_display_manage = gr.HTML(label="📄 Manage Bookmarks Display") # Upload and Process Bookmarks Tab with gr.Tab("Upload and Process Bookmarks"): gr.Markdown(""" ## 📂 **Upload and Process Bookmarks** ### 📝 **Steps to Upload and Process:** 1. **🔽 Upload Bookmarks File:** - Click on the **"📁 Upload Bookmarks HTML File"** button. - Select your browser's exported bookmarks HTML file from your device. 2. **⚙️ Process Bookmarks:** - After uploading, click on the **"⚙️ Process Bookmarks"** button. - SmartMarks will parse your bookmarks, fetch additional information, generate summaries, and categorize each link based on predefined categories. 3. **📄 View Processed Bookmarks:** - Once processing is complete, your bookmarks will be displayed in an organized and visually appealing format below. """) upload = gr.File(label="📁 Upload Bookmarks HTML File", type='binary') process_button = gr.Button("⚙️ Process Bookmarks") output_text = gr.Textbox(label="✅ Output", interactive=False) bookmark_display = gr.HTML(label="📄 Bookmarks") process_button.click( process_uploaded_file, inputs=[upload, state_bookmarks], outputs=[output_text, bookmark_display, bookmark_selector, bookmark_display_manage, state_bookmarks] ) # Chat with Bookmarks Tab with gr.Tab("Chat with Bookmarks"): gr.Markdown(""" ## 💬 **Chat with Bookmarks** ### 🤖 **How to Interact:** 1. **✍️ Enter Your Query:** - In the **"✍️ Ask about your bookmarks"** textbox, type your question or keyword related to your bookmarks. For example, "Do I have any bookmarks about GenerativeAI?" 2. **📨 Submit Your Query:** - Click the **"📨 Send"** button to submit your query. 3. **📈 Receive AI-Driven Responses:** - SmartMarks will analyze your query and provide relevant bookmarks that match your request, making it easier to find specific links without manual searching. 4. **🗂️ View Chat History:** - All your queries and the corresponding AI responses are displayed in the chat history for your reference. """) with gr.Row(): chat_history_display = gr.Chatbot(label="🗨️ Chat History") with gr.Column(scale=1): chat_input = gr.Textbox( label="✍️ Ask about your bookmarks", placeholder="e.g., Do I have any bookmarks about GenerativeAI?", lines=1, interactive=True ) chat_button = gr.Button("📨 Send") # When user presses Enter in chat_input chat_input.submit( chatbot_response, inputs=[chat_input, state_bookmarks], outputs=chat_history_display ) # When user clicks Send button chat_button.click( chatbot_response, inputs=[chat_input, state_bookmarks], outputs=chat_history_display ) # Manage Bookmarks Tab with gr.Tab("Manage Bookmarks"): gr.Markdown(""" ## 🛠️ **Manage Bookmarks** ### 🗂️ **Features:** 1. **👁️ View Bookmarks:** - All your processed bookmarks are displayed here with their respective categories and summaries. 2. **✅ Select Bookmarks:** - Use the checkboxes next to each bookmark to select one, multiple, or all bookmarks you wish to manage. 3. **🗑️ Delete Selected Bookmarks:** - After selecting the desired bookmarks, click the **"🗑️ Delete Selected Bookmarks"** button to remove them from your list. 4. **✏️ Edit Categories:** - Select the bookmarks you want to re-categorize. - Choose a new category from the dropdown menu labeled **"🆕 New Category"**. - Click the **"✏️ Edit Category of Selected Bookmarks"** button to update their categories. 5. **💾 Export Bookmarks:** - Click the **"💾 Export Bookmarks"** button to download your updated bookmarks as an HTML file. - This file can be uploaded back to your browser to reflect the changes made within SmartMarks. 6. **🔄 Refresh Bookmarks:** - Click the **"🔄 Refresh Bookmarks"** button to ensure the latest state is reflected in the display. """) manage_output = gr.Textbox(label="🔄 Manage Output", interactive=False) new_category_input = gr.Dropdown(label="🆕 New Category", choices=CATEGORIES, value="Uncategorized") with gr.Row(): delete_button = gr.Button("🗑️ Delete Selected Bookmarks") edit_category_button = gr.Button("✏️ Edit Category of Selected Bookmarks") export_button = gr.Button("💾 Export Bookmarks") download_link = gr.HTML(label="📥 Download Exported Bookmarks") refresh_button = gr.Button("🔄 Refresh Bookmarks") # Define button actions delete_button.click( delete_selected_bookmarks, inputs=[bookmark_selector, state_bookmarks], outputs=[manage_output, bookmark_selector, bookmark_display_manage, state_bookmarks] ) edit_category_button.click( edit_selected_bookmarks_category, inputs=[bookmark_selector, new_category_input, state_bookmarks], outputs=[manage_output, bookmark_selector, bookmark_display_manage, state_bookmarks] ) export_button.click( export_bookmarks, inputs=[state_bookmarks], outputs=download_link ) refresh_button.click( lambda bookmarks: ( [ f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" for i, bookmark in enumerate(bookmarks) ], display_bookmarks() ), inputs=[state_bookmarks], outputs=[bookmark_selector, bookmark_display_manage] ) logger.info("Launching Gradio app") demo.launch(debug=True) except Exception as e: logger.error(f"Error building the app: {e}") print(f"Error building the app: {e}") if __name__ == "__main__": build_app()