app.py

# app.py

import gradio as gr
from bs4 import BeautifulSoup
import requests
from sentence_transformers import SentenceTransformer
import faiss
import numpy as np
import asyncio
import aiohttp
import re
import pandas as pd

# Initialize models and variables
embedding_model = SentenceTransformer('all-MiniLM-L6-v2')
faiss_index = None
bookmarks = []
fetch_cache = {}

# Define the categories
CATEGORIES = [
    "Social Media",
    "News and Media",
    "Education and Learning",
    "Entertainment",
    "Shopping and E-commerce",
    "Finance and Banking",
    "Technology",
    "Health and Fitness",
    "Travel and Tourism",
    "Food and Recipes",
    "Sports",
    "Arts and Culture",
    "Government and Politics",
    "Business and Economy",
    "Science and Research",
    "Personal Blogs and Journals",
    "Job Search and Careers",
    "Music and Audio",
    "Videos and Movies",
    "Reference and Knowledge Bases",
    "Dead Link",
    "Uncategorized",
]

def parse_bookmarks(file_content):
    soup = BeautifulSoup(file_content, 'html.parser')
    extracted_bookmarks = []
    for link in soup.find_all('a'):
        url = link.get('href')
        title = link.text.strip()
        if url and title:
            extracted_bookmarks.append({'url': url, 'title': title})
    return extracted_bookmarks

async def fetch_url_info(session, bookmark):
    url = bookmark['url']
    if url in fetch_cache:
        bookmark.update(fetch_cache[url])
        return bookmark

    try:
        async with session.get(url, timeout=5) as response:
            bookmark['etag'] = response.headers.get('ETag', 'N/A')
            bookmark['status_code'] = response.status

            if response.status >= 400:
                bookmark['dead_link'] = True
                bookmark['description'] = ''
            else:
                bookmark['dead_link'] = False
                content = await response.text()
                soup = BeautifulSoup(content, 'html.parser')

                # Extract meta description or Open Graph description
                meta_description = soup.find('meta', attrs={'name': 'description'})
                og_description = soup.find('meta', attrs={'property': 'og:description'})
                if og_description and og_description.get('content'):
                    description = og_description.get('content')
                elif meta_description and meta_description.get('content'):
                    description = meta_description.get('content')
                else:
                    description = ''

                bookmark['description'] = description
    except Exception as e:
        bookmark['dead_link'] = True
        bookmark['etag'] = 'N/A'
        bookmark['status_code'] = 'N/A'
        bookmark['description'] = ''
    finally:
        fetch_cache[url] = {
            'etag': bookmark.get('etag'),
            'status_code': bookmark.get('status_code'),
            'dead_link': bookmark.get('dead_link'),
            'description': bookmark.get('description'),
        }
    return bookmark

async def process_bookmarks_async(bookmarks):
    async with aiohttp.ClientSession() as session:
        tasks = []
        for bookmark in bookmarks:
            task = asyncio.ensure_future(fetch_url_info(session, bookmark))
            tasks.append(task)
        await asyncio.gather(*tasks)

def generate_summary(bookmark):
    description = bookmark.get('description', '')
    if description:
        bookmark['summary'] = description
    else:
        title = bookmark.get('title', '')
        if title:
            bookmark['summary'] = title
        else:
            bookmark['summary'] = 'No summary available.'
    return bookmark

def assign_category(bookmark):
    if bookmark.get('dead_link'):
        bookmark['category'] = 'Dead Link'
        return bookmark

    summary = bookmark.get('summary', '').lower()
    assigned_category = 'Uncategorized'

    # Keywords associated with each category
    category_keywords = {
        "Social Media": ["social media", "networking", "friends", "connect", "posts", "profile"],
        "News and Media": ["news", "journalism", "media", "headlines", "breaking news"],
        "Education and Learning": ["education", "learning", "courses", "tutorial", "university", "academy", "study"],
        "Entertainment": ["entertainment", "movies", "tv shows", "games", "comics", "fun"],
        "Shopping and E-commerce": ["shopping", "e-commerce", "buy", "sell", "marketplace", "deals", "store"],
        "Finance and Banking": ["finance", "banking", "investment", "money", "economy", "stock", "trading"],
        "Technology": ["technology", "tech", "gadgets", "software", "computers", "innovation"],
        "Health and Fitness": ["health", "fitness", "medical", "wellness", "exercise", "diet"],
        "Travel and Tourism": ["travel", "tourism", "destinations", "hotels", "flights", "vacation"],
        "Food and Recipes": ["food", "recipes", "cooking", "cuisine", "restaurant", "dining"],
        "Sports": ["sports", "scores", "teams", "athletics", "matches", "leagues"],
        "Arts and Culture": ["arts", "culture", "museum", "gallery", "exhibition", "artistic"],
        "Government and Politics": ["government", "politics", "policy", "election", "public service"],
        "Business and Economy": ["business", "corporate", "industry", "economy", "markets"],
        "Science and Research": ["science", "research", "experiment", "laboratory", "study", "scientific"],
        "Personal Blogs and Journals": ["blog", "journal", "personal", "diary", "thoughts", "opinions"],
        "Job Search and Careers": ["jobs", "careers", "recruitment", "resume", "employment", "hiring"],
        "Music and Audio": ["music", "audio", "songs", "albums", "artists", "bands"],
        "Videos and Movies": ["video", "movies", "film", "clips", "trailers", "cinema"],
        "Reference and Knowledge Bases": ["reference", "encyclopedia", "dictionary", "wiki", "knowledge", "information"],
    }

    for category, keywords in category_keywords.items():
        for keyword in keywords:
            if re.search(r'\b' + re.escape(keyword) + r'\b', summary):
                assigned_category = category
                break
        if assigned_category != 'Uncategorized':
            break

    bookmark['category'] = assigned_category
    return bookmark

def vectorize_and_index(bookmarks):
    summaries = [bookmark['summary'] for bookmark in bookmarks]
    embeddings = embedding_model.encode(summaries)
    dimension = embeddings.shape[1]
    faiss_idx = faiss.IndexFlatL2(dimension)
    faiss_idx.add(np.array(embeddings))
    return faiss_idx, embeddings

def bookmarks_to_dataframe():
    data = []
    for i, bookmark in enumerate(bookmarks):
        index = i + 1
        status = "Dead Link" if bookmark.get('dead_link') else "Active"
        data.append({
            'Index': index,
            'Title': bookmark['title'],
            'URL': bookmark['url'],
            'Category': bookmark.get('category', 'Uncategorized'),
            'Status': status,
            'Summary': bookmark.get('summary', ''),
        })
    df = pd.DataFrame(data)
    return df

def process_uploaded_file(file):
    global bookmarks, faiss_index
    if file is None:
        return "Please upload a bookmarks HTML file.", pd.DataFrame()
    try:
        file_content = file.decode('utf-8')
    except UnicodeDecodeError:
        return "Error decoding the file. Please ensure it's a valid HTML file.", pd.DataFrame()

    bookmarks = parse_bookmarks(file_content)

    if not bookmarks:
        return "No bookmarks found in the uploaded file.", pd.DataFrame()

    # Asynchronously fetch bookmark info
    asyncio.run(process_bookmarks_async(bookmarks))

    # Generate summaries and assign categories
    for bookmark in bookmarks:
        generate_summary(bookmark)
        assign_category(bookmark)

    faiss_index, embeddings = vectorize_and_index(bookmarks)
    message = f"Successfully processed {len(bookmarks)} bookmarks."
    bookmark_df = bookmarks_to_dataframe()
    return message, bookmark_df

def chatbot_response(user_query):
    if faiss_index is None or not bookmarks:
        return "No bookmarks available. Please upload and process your bookmarks first."

    # Vectorize user query
    user_embedding = embedding_model.encode([user_query])
    D, I = faiss_index.search(np.array(user_embedding), k=5)  # Retrieve top 5 matches

    # Generate response
    response = ""
    for idx in I[0]:
        if idx < len(bookmarks):
            bookmark = bookmarks[idx]
            index = idx + 1  # Start index at 1
            response += f"{index}. Title: {bookmark['title']}\nURL: {bookmark['url']}\nCategory: {bookmark.get('category', 'Uncategorized')}\nSummary: {bookmark['summary']}\n\n"
    return response.strip()

def edit_bookmark(row):
    global faiss_index
    try:
        bookmark_idx = int(row['Index']) - 1  # Adjust index to match list (starting at 0)
        if bookmark_idx < 0 or bookmark_idx >= len(bookmarks):
            return "Invalid bookmark index.", bookmarks_to_dataframe()
        bookmarks[bookmark_idx]['title'] = row['Title']
        bookmarks[bookmark_idx]['url'] = row['URL']
        bookmarks[bookmark_idx]['category'] = row['Category']
        # Re-fetch bookmark info
        asyncio.run(process_bookmarks_async([bookmarks[bookmark_idx]]))
        generate_summary(bookmarks[bookmark_idx])
        # Rebuild the FAISS index
        faiss_index, embeddings = vectorize_and_index(bookmarks)
        message = "Bookmark updated successfully."
        updated_df = bookmarks_to_dataframe()
        return message, updated_df
    except Exception as e:
        return f"Error: {str(e)}", bookmarks_to_dataframe()

def delete_bookmarks(selected_indices):
    global faiss_index
    try:
        indices = sorted([int(idx) - 1 for idx in selected_indices], reverse=True)
        for idx in indices:
            if 0 <= idx < len(bookmarks):
                bookmarks.pop(idx)
        # Rebuild the FAISS index
        if bookmarks:
            faiss_index, embeddings = vectorize_and_index(bookmarks)
        else:
            faiss_index = None
        message = "Selected bookmarks deleted successfully."
        updated_df = bookmarks_to_dataframe()
        return message, updated_df
    except Exception as e:
        return f"Error: {str(e)}", bookmarks_to_dataframe()

def export_bookmarks():
    if not bookmarks:
        return None
    # Create an HTML content similar to the imported bookmarks file
    soup = BeautifulSoup("<!DOCTYPE NETSCAPE-Bookmark-file-1><Title>Bookmarks</Title><H1>Bookmarks</H1>", 'html.parser')
    dl = soup.new_tag('DL')
    for bookmark in bookmarks:
        dt = soup.new_tag('DT')
        a = soup.new_tag('A', href=bookmark['url'])
        a.string = bookmark['title']
        dt.append(a)
        dl.append(dt)
    soup.append(dl)
    html_content = str(soup)
    return html_content

def build_app():
    with gr.Blocks(css="app.css") as demo:
        gr.Markdown("<h1>Bookmark Manager App</h1>")

        with gr.Tab("Upload and Process Bookmarks"):
            upload = gr.File(label="Upload Bookmarks HTML File", type='binary')
            process_button = gr.Button("Process Bookmarks")
            output_text = gr.Textbox(label="Output")
            bookmark_table = gr.Dataframe(label="Bookmarks", interactive=False)

            def update_bookmark_table(file):
                message, df = process_uploaded_file(file)
                return message, df

            process_button.click(
                update_bookmark_table,
                inputs=upload,
                outputs=[output_text, bookmark_table]
            )

        with gr.Tab("Chat with Bookmarks"):
            user_input = gr.Textbox(label="Ask about your bookmarks")
            chat_output = gr.Textbox(label="Chatbot Response")
            chat_button = gr.Button("Send")

            chat_button.click(
                chatbot_response,
                inputs=user_input,
                outputs=chat_output
            )

        with gr.Tab("Manage Bookmarks"):
            manage_output = gr.Textbox(label="Manage Output")
            bookmark_table_manage = gr.Dataframe(label="Bookmarks", interactive=True)
            selected_indices = gr.Textbox(label="Selected Indices (comma-separated)", visible=False)
            delete_button = gr.Button("Delete Selected Bookmarks")
            export_button = gr.Button("Export Bookmarks")
            download_link = gr.File(label="Download Exported Bookmarks", interactive=False)

            def update_manage_table():
                df = bookmarks_to_dataframe()
                return df

            def delete_selected_bookmarks(dataframe):
                selected_indices = dataframe['Index'].tolist()
                message, updated_df = delete_bookmarks(selected_indices)
                return message, updated_df

            def export_bookmarks_file():
                content = export_bookmarks()
                if content:
                    with open('bookmarks.html', 'w', encoding='utf-8') as f:
                        f.write(content)
                    return 'bookmarks.html'
                else:
                    return None

            bookmark_table_manage.change(
                edit_bookmark,
                inputs=bookmark_table_manage,
                outputs=[manage_output, bookmark_table_manage]
            )

            delete_button.click(
                delete_selected_bookmarks,
                inputs=bookmark_table_manage,
                outputs=[manage_output, bookmark_table_manage]
            )

            export_button.click(
                export_bookmarks_file,
                inputs=None,
                outputs=download_link
            )

            # Initial load of the bookmarks table
            bookmark_table_manage.value = update_manage_table()

    demo.launch()

if __name__ == "__main__":
    build_app()