Spaces:
Running
Running
| # app.py | |
| import gradio as gr | |
| from bs4 import BeautifulSoup | |
| import requests | |
| from sentence_transformers import SentenceTransformer | |
| import faiss | |
| import numpy as np | |
| import asyncio | |
| import aiohttp | |
| import re | |
| import base64 | |
| # Initialize models and variables | |
| embedding_model = SentenceTransformer('all-MiniLM-L6-v2') | |
| faiss_index = None | |
| bookmarks = [] | |
| fetch_cache = {} | |
| # Define the categories | |
| CATEGORIES = [ | |
| "Social Media", | |
| "News and Media", | |
| "Education and Learning", | |
| "Entertainment", | |
| "Shopping and E-commerce", | |
| "Finance and Banking", | |
| "Technology", | |
| "Health and Fitness", | |
| "Travel and Tourism", | |
| "Food and Recipes", | |
| "Sports", | |
| "Arts and Culture", | |
| "Government and Politics", | |
| "Business and Economy", | |
| "Science and Research", | |
| "Personal Blogs and Journals", | |
| "Job Search and Careers", | |
| "Music and Audio", | |
| "Videos and Movies", | |
| "Reference and Knowledge Bases", | |
| "Dead Link", | |
| "Uncategorized", | |
| ] | |
| def parse_bookmarks(file_content): | |
| soup = BeautifulSoup(file_content, 'html.parser') | |
| extracted_bookmarks = [] | |
| for link in soup.find_all('a'): | |
| url = link.get('href') | |
| title = link.text.strip() | |
| if url and title: | |
| extracted_bookmarks.append({'url': url, 'title': title}) | |
| return extracted_bookmarks | |
| async def fetch_url_info(session, bookmark): | |
| url = bookmark['url'] | |
| if url in fetch_cache: | |
| bookmark.update(fetch_cache[url]) | |
| return bookmark | |
| try: | |
| async with session.get(url, timeout=5) as response: | |
| bookmark['etag'] = response.headers.get('ETag', 'N/A') | |
| bookmark['status_code'] = response.status | |
| if response.status >= 400: | |
| bookmark['dead_link'] = True | |
| bookmark['description'] = '' | |
| else: | |
| bookmark['dead_link'] = False | |
| content = await response.text() | |
| soup = BeautifulSoup(content, 'html.parser') | |
| # Extract meta description or Open Graph description | |
| meta_description = soup.find('meta', attrs={'name': 'description'}) | |
| og_description = soup.find('meta', attrs={'property': 'og:description'}) | |
| if og_description and og_description.get('content'): | |
| description = og_description.get('content') | |
| elif meta_description and meta_description.get('content'): | |
| description = meta_description.get('content') | |
| else: | |
| description = '' | |
| bookmark['description'] = description | |
| except Exception as e: | |
| bookmark['dead_link'] = True | |
| bookmark['etag'] = 'N/A' | |
| bookmark['status_code'] = 'N/A' | |
| bookmark['description'] = '' | |
| finally: | |
| fetch_cache[url] = { | |
| 'etag': bookmark.get('etag'), | |
| 'status_code': bookmark.get('status_code'), | |
| 'dead_link': bookmark.get('dead_link'), | |
| 'description': bookmark.get('description'), | |
| } | |
| return bookmark | |
| async def process_bookmarks_async(bookmarks): | |
| async with aiohttp.ClientSession() as session: | |
| tasks = [] | |
| for bookmark in bookmarks: | |
| task = asyncio.ensure_future(fetch_url_info(session, bookmark)) | |
| tasks.append(task) | |
| await asyncio.gather(*tasks) | |
| def generate_summary(bookmark): | |
| description = bookmark.get('description', '') | |
| if description: | |
| bookmark['summary'] = description | |
| else: | |
| title = bookmark.get('title', '') | |
| if title: | |
| bookmark['summary'] = title | |
| else: | |
| bookmark['summary'] = 'No summary available.' | |
| return bookmark | |
| def assign_category(bookmark): | |
| if bookmark.get('dead_link'): | |
| bookmark['category'] = 'Dead Link' | |
| return bookmark | |
| summary = bookmark.get('summary', '').lower() | |
| assigned_category = 'Uncategorized' | |
| # Keywords associated with each category | |
| category_keywords = { | |
| "Social Media": ["social media", "networking", "friends", "connect", "posts", "profile"], | |
| "News and Media": ["news", "journalism", "media", "headlines", "breaking news"], | |
| "Education and Learning": ["education", "learning", "courses", "tutorial", "university", "academy", "study"], | |
| "Entertainment": ["entertainment", "movies", "tv shows", "games", "comics", "fun"], | |
| "Shopping and E-commerce": ["shopping", "e-commerce", "buy", "sell", "marketplace", "deals", "store"], | |
| "Finance and Banking": ["finance", "banking", "investment", "money", "economy", "stock", "trading"], | |
| "Technology": ["technology", "tech", "gadgets", "software", "computers", "innovation"], | |
| "Health and Fitness": ["health", "fitness", "medical", "wellness", "exercise", "diet"], | |
| "Travel and Tourism": ["travel", "tourism", "destinations", "hotels", "flights", "vacation"], | |
| "Food and Recipes": ["food", "recipes", "cooking", "cuisine", "restaurant", "dining"], | |
| "Sports": ["sports", "scores", "teams", "athletics", "matches", "leagues"], | |
| "Arts and Culture": ["arts", "culture", "museum", "gallery", "exhibition", "artistic"], | |
| "Government and Politics": ["government", "politics", "policy", "election", "public service"], | |
| "Business and Economy": ["business", "corporate", "industry", "economy", "markets"], | |
| "Science and Research": ["science", "research", "experiment", "laboratory", "study", "scientific"], | |
| "Personal Blogs and Journals": ["blog", "journal", "personal", "diary", "thoughts", "opinions"], | |
| "Job Search and Careers": ["jobs", "careers", "recruitment", "resume", "employment", "hiring"], | |
| "Music and Audio": ["music", "audio", "songs", "albums", "artists", "bands"], | |
| "Videos and Movies": ["video", "movies", "film", "clips", "trailers", "cinema"], | |
| "Reference and Knowledge Bases": ["reference", "encyclopedia", "dictionary", "wiki", "knowledge", "information"], | |
| } | |
| for category, keywords in category_keywords.items(): | |
| for keyword in keywords: | |
| if re.search(r'\b' + re.escape(keyword) + r'\b', summary): | |
| assigned_category = category | |
| break | |
| if assigned_category != 'Uncategorized': | |
| break | |
| bookmark['category'] = assigned_category | |
| return bookmark | |
| def vectorize_and_index(bookmarks): | |
| summaries = [bookmark['summary'] for bookmark in bookmarks] | |
| embeddings = embedding_model.encode(summaries) | |
| dimension = embeddings.shape[1] | |
| faiss_idx = faiss.IndexFlatL2(dimension) | |
| faiss_idx.add(np.array(embeddings)) | |
| return faiss_idx, embeddings | |
| def display_bookmarks(): | |
| cards = '' | |
| for i, bookmark in enumerate(bookmarks): | |
| index = i + 1 # Start index at 1 | |
| status = "Dead Link" if bookmark.get('dead_link') else "Active" | |
| title = bookmark['title'] | |
| url = bookmark['url'] | |
| etag = bookmark.get('etag', 'N/A') | |
| summary = bookmark.get('summary', '') | |
| category = bookmark.get('category', 'Uncategorized') | |
| # Apply inline styles for dead links | |
| if bookmark.get('dead_link'): | |
| card_style = "border: 2px solid #D32F2F;" | |
| text_style = "color: #D32F2F;" | |
| else: | |
| card_style = "" | |
| text_style = "" | |
| card_html = f''' | |
| <div class="card" style="{card_style}"> | |
| <div class="card-content"> | |
| <h3 style="{text_style}">{index}. {title}</h3> | |
| <p style="{text_style}"><strong>Category:</strong> {category}</p> | |
| <p style="{text_style}"><strong>URL:</strong> <a href="{url}" target="_blank" style="{text_style}">{url}</a></p> | |
| <p style="{text_style}"><strong>Status:</strong> {status}</p> | |
| <p style="{text_style}"><strong>ETag:</strong> {etag}</p> | |
| <p style="{text_style}"><strong>Summary:</strong> {summary}</p> | |
| </div> | |
| </div> | |
| ''' | |
| cards += card_html | |
| return cards | |
| def process_uploaded_file(file): | |
| global bookmarks, faiss_index | |
| if file is None: | |
| return "Please upload a bookmarks HTML file.", '' | |
| try: | |
| file_content = file.decode('utf-8') | |
| except UnicodeDecodeError: | |
| return "Error decoding the file. Please ensure it's a valid HTML file.", '' | |
| bookmarks = parse_bookmarks(file_content) | |
| if not bookmarks: | |
| return "No bookmarks found in the uploaded file.", '' | |
| # Asynchronously fetch bookmark info | |
| asyncio.run(process_bookmarks_async(bookmarks)) | |
| # Generate summaries and assign categories | |
| for bookmark in bookmarks: | |
| generate_summary(bookmark) | |
| assign_category(bookmark) | |
| faiss_index, embeddings = vectorize_and_index(bookmarks) | |
| message = f"Successfully processed {len(bookmarks)} bookmarks." | |
| bookmark_html = display_bookmarks() | |
| return message, bookmark_html | |
| def chatbot_response(user_query): | |
| if faiss_index is None or not bookmarks: | |
| return "No bookmarks available. Please upload and process your bookmarks first." | |
| # Vectorize user query | |
| user_embedding = embedding_model.encode([user_query]) | |
| D, I = faiss_index.search(np.array(user_embedding), k=5) # Retrieve top 5 matches | |
| # Generate response | |
| response = "" | |
| for idx in I[0]: | |
| if idx < len(bookmarks): | |
| bookmark = bookmarks[idx] | |
| index = idx + 1 # Start index at 1 | |
| response += f"{index}. Title: {bookmark['title']}\nURL: {bookmark['url']}\nCategory: {bookmark.get('category', 'Uncategorized')}\nSummary: {bookmark['summary']}\n\n" | |
| return response.strip() | |
| def edit_bookmark(bookmark_idx, new_title, new_url, new_category): | |
| global faiss_index | |
| try: | |
| bookmark_idx = int(bookmark_idx) - 1 # Adjust index to match list (starting at 0) | |
| if bookmark_idx < 0 or bookmark_idx >= len(bookmarks): | |
| return "Invalid bookmark index.", display_bookmarks() | |
| bookmarks[bookmark_idx]['title'] = new_title | |
| bookmarks[bookmark_idx]['url'] = new_url | |
| bookmarks[bookmark_idx]['category'] = new_category | |
| # Re-fetch bookmark info | |
| asyncio.run(process_bookmarks_async([bookmarks[bookmark_idx]])) | |
| generate_summary(bookmarks[bookmark_idx]) | |
| # Rebuild the FAISS index | |
| faiss_index, embeddings = vectorize_and_index(bookmarks) | |
| message = "Bookmark updated successfully." | |
| updated_html = display_bookmarks() | |
| return message, updated_html | |
| except Exception as e: | |
| return f"Error: {str(e)}", display_bookmarks() | |
| def delete_bookmarks(indices_str): | |
| global faiss_index | |
| try: | |
| indices = [int(idx.strip()) - 1 for idx in indices_str.split(',') if idx.strip().isdigit()] | |
| indices = sorted(indices, reverse=True) | |
| for idx in indices: | |
| if 0 <= idx < len(bookmarks): | |
| bookmarks.pop(idx) | |
| # Rebuild the FAISS index | |
| if bookmarks: | |
| faiss_index, embeddings = vectorize_and_index(bookmarks) | |
| else: | |
| faiss_index = None | |
| message = "Selected bookmarks deleted successfully." | |
| updated_html = display_bookmarks() | |
| return message, updated_html | |
| except Exception as e: | |
| return f"Error: {str(e)}", display_bookmarks() | |
| def export_bookmarks(): | |
| if not bookmarks: | |
| return None | |
| # Create an HTML content similar to the imported bookmarks file | |
| soup = BeautifulSoup("<!DOCTYPE NETSCAPE-Bookmark-file-1><Title>Bookmarks</Title><H1>Bookmarks</H1>", 'html.parser') | |
| dl = soup.new_tag('DL') | |
| for bookmark in bookmarks: | |
| dt = soup.new_tag('DT') | |
| a = soup.new_tag('A', href=bookmark['url']) | |
| a.string = bookmark['title'] | |
| dt.append(a) | |
| dl.append(dt) | |
| soup.append(dl) | |
| html_content = str(soup) | |
| # Encode the HTML content to base64 for download | |
| b64 = base64.b64encode(html_content.encode()).decode() | |
| href = f'data:text/html;base64,{b64}' | |
| return href | |
| def build_app(): | |
| with gr.Blocks(css="app.css") as demo: | |
| gr.Markdown("<h1>Bookmark Manager App</h1>") | |
| with gr.Tab("Upload and Process Bookmarks"): | |
| upload = gr.File(label="Upload Bookmarks HTML File", type='binary') | |
| process_button = gr.Button("Process Bookmarks") | |
| output_text = gr.Textbox(label="Output") | |
| bookmark_display = gr.HTML(label="Bookmarks") | |
| def update_bookmark_display(file): | |
| message, html_content = process_uploaded_file(file) | |
| return message, html_content | |
| process_button.click( | |
| update_bookmark_display, | |
| inputs=upload, | |
| outputs=[output_text, bookmark_display] | |
| ) | |
| with gr.Tab("Chat with Bookmarks"): | |
| user_input = gr.Textbox(label="Ask about your bookmarks") | |
| chat_output = gr.Textbox(label="Chatbot Response") | |
| chat_button = gr.Button("Send") | |
| chat_button.click( | |
| chatbot_response, | |
| inputs=user_input, | |
| outputs=chat_output | |
| ) | |
| with gr.Tab("Manage Bookmarks"): | |
| manage_output = gr.Textbox(label="Manage Output") | |
| bookmark_display_manage = gr.HTML(label="Bookmarks") | |
| refresh_button = gr.Button("Refresh Bookmark List") | |
| indices_input = gr.Textbox(label="Bookmark Indices to Delete (comma-separated)") | |
| delete_button = gr.Button("Delete Selected Bookmarks") | |
| export_button = gr.Button("Export Bookmarks") | |
| download_link = gr.HTML(label="Download Exported Bookmarks") | |
| with gr.Row(): | |
| index_input = gr.Number(label="Bookmark Index (Starting from 1)", precision=0) | |
| new_title_input = gr.Textbox(label="New Title") | |
| new_url_input = gr.Textbox(label="New URL") | |
| new_category_input = gr.Dropdown(label="New Category", choices=CATEGORIES) | |
| edit_button = gr.Button("Edit Bookmark") | |
| def update_manage_display(): | |
| html_content = display_bookmarks() | |
| return html_content | |
| refresh_button.click( | |
| update_manage_display, | |
| inputs=None, | |
| outputs=bookmark_display_manage | |
| ) | |
| edit_button.click( | |
| edit_bookmark, | |
| inputs=[index_input, new_title_input, new_url_input, new_category_input], | |
| outputs=[manage_output, bookmark_display_manage] | |
| ) | |
| delete_button.click( | |
| delete_bookmarks, | |
| inputs=indices_input, | |
| outputs=[manage_output, bookmark_display_manage] | |
| ) | |
| def provide_download_link(): | |
| href = export_bookmarks() | |
| if href: | |
| return f'<a href="{href}" download="bookmarks.html">Download Exported Bookmarks</a>' | |
| else: | |
| return "No bookmarks to export." | |
| export_button.click( | |
| provide_download_link, | |
| inputs=None, | |
| outputs=download_link | |
| ) | |
| # Initial load of the bookmarks display | |
| bookmark_display_manage.value = update_manage_display() | |
| demo.launch() | |
| if __name__ == "__main__": | |
| build_app() | |