app.py

import streamlit as st
st.set_page_config(page_title="Advanced File Downloader", layout="wide")

# Core imports
import os
import subprocess
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError
import asyncio
import logging
from urllib.parse import urlparse
import re
from pathlib import Path
from io import BytesIO
import random
from bs4 import BeautifulSoup
from PyPDF2 import PdfReader
import zipfile
import tempfile
import mimetypes
import requests
import datetime
import spacy
import spacy.cli
from spacy.language import Language
import google_auth_oauthlib.flow
import googleapiclient.discovery
import google.auth.transport.requests
from async_timeout import timeout as async_timeout
import pandas as pd
from sentence_transformers import SentenceTransformer
from transformers import pipeline
import schedule
import threading
import time
import hashlib
from reportlab.lib.pagesizes import letter
from reportlab.pdfgen import canvas
from sklearn.cluster import KMeans
import numpy as np
import base64
# -------------------- Logging Setup --------------------
logging.basicConfig(
    filename='advanced_download_log.txt',
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

GOOGLE_OAUTH_CONFIG = {
    "web": {
        "client_id": "90798824947-u25obg1q844qeikjoh4jdmi579kn9p1c.apps.googleusercontent.com",
        "project_id": "huggingface-449214",
        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
        "token_uri": "https://oauth2.googleapis.com/token",
        "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
        "client_secret": "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",
        "redirect_uris": ["https://euler314-craw-web.hf.space/"]
    }
}

# Playwright Setup
def install_playwright_dependencies():
    os.environ['PLAYWRIGHT_BROWSERS_PATH'] = os.path.expanduser("~/.cache/ms-playwright")
    subprocess.run(['apt-get', 'update', '-y'], check=True)
    packages = [
        'libnss3', 'libnss3-tools', 'libnspr4', 'libatk1.0-0',
        'libatk-bridge2.0-0', 'libatspi2.0-0', 'libcups2', 'libxcomposite1',
        'libxdamage1', 'libdrm2', 'libgbm1', 'libpango-1.0-0'
    ]
    subprocess.run(['apt-get', 'install', '-y', '--no-install-recommends'] + packages, check=True)
    subprocess.run(['python3', '-m', 'playwright', 'install', 'chromium'], check=True)

install_playwright_dependencies()

# Model Loading
@st.cache_resource
def load_models():
    try:
        # Load spaCy model
        try:
            nlp = spacy.load("en_core_web_sm")
        except OSError:
            st.info("Downloading spaCy model...")
            spacy.cli.download("en_core_web_sm")
            nlp = spacy.load("en_core_web_sm")

        # Load SentenceTransformer
        try:
            semantic_model = SentenceTransformer('deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B')
        except Exception as e:
            st.error(f"Error loading SentenceTransformer: {e}")
            semantic_model = None

        # Load Transformers pipeline
        try:
            summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
        except Exception as e:
            st.error(f"Error loading Transformers: {e}")
            summarizer = None

        return nlp, semantic_model, summarizer
    except Exception as e:
        st.error(f"Error loading models: {e}")
        return None, None, None

nlp_model, semantic_model, summarizer = load_models()

# Utility Functions
def get_random_user_agent():
    USER_AGENTS = [
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15',
        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0',
    ]
    return random.choice(USER_AGENTS)

def sizeof_fmt(num, suffix='B'):
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if abs(num) < 1024.0:
            return f"{num:3.1f}{unit}{suffix}"
        num /= 1024.0
    return f"{num:.1f}Y{suffix}"

def create_zip_file(file_paths, output_dir):
    timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
    zip_path = os.path.join(output_dir, f"downloads_{timestamp}.zip")
    with zipfile.ZipFile(zip_path, 'w') as zipf:
        for file_path in file_paths:
            zipf.write(file_path, os.path.basename(file_path))
    return zip_path

# Google Drive Functions
def get_google_auth_url():
    client_config = GOOGLE_OAUTH_CONFIG["web"]
    flow = google_auth_oauthlib.flow.Flow.from_client_config(
        {"web": client_config},
        scopes=["https://www.googleapis.com/auth/drive.file"]
    )
    flow.redirect_uri = client_config["redirect_uris"][0]
    authorization_url, _ = flow.authorization_url(
        access_type="offline",
        include_granted_scopes="true",
        prompt="consent"
    )
    return authorization_url

def exchange_code_for_credentials(auth_code):
    if not auth_code.strip():
        return None, "No code provided."
    try:
        client_config = GOOGLE_OAUTH_CONFIG["web"]
        flow = google_auth_oauthlib.flow.Flow.from_client_config(
            {"web": client_config},
            scopes=["https://www.googleapis.com/auth/drive.file"]
        )
        flow.redirect_uri = client_config["redirect_uris"][0]
        flow.fetch_token(code=auth_code.strip())
        creds = flow.credentials
        if not creds or not creds.valid:
            return None, "Could not validate credentials. Check code and try again."
        return creds, "Google Sign-In successful!"
    except Exception as e:
        return None, f"Error during token exchange: {e}"

def google_drive_upload(file_path, credentials, folder_id=None):
    try:
        drive_service = googleapiclient.discovery.build("drive", "v3", credentials=credentials)
        file_metadata = {'name': os.path.basename(file_path)}
        if folder_id:
            file_metadata['parents'] = [folder_id]
        media = googleapiclient.http.MediaFileUpload(file_path, resumable=True)
        created = drive_service.files().create(body=file_metadata, media_body=media, fields='id').execute()
        return created.get("id", "")
    except Exception as e:
        return f"Error uploading to Drive: {str(e)}"

def create_drive_folder(drive_service, name):
    folder_metadata = {'name': name, 'mimeType': 'application/vnd.google-apps.folder'}
    folder = drive_service.files().create(body=folder_metadata, fields='id').execute()
    return folder.get('id')

# DownloadManager Class
class DownloadManager:
    def __init__(self, use_proxy=False, proxy=None, query=None, num_results=5):
        self.use_proxy = use_proxy
        self.proxy = proxy
        self.query = query
        self.num_results = num_results
        self.playwright = None
        self.browser = None
        self.context = None
        self.page = None

    async def __aenter__(self):
        self.playwright = await async_playwright().start()
        opts = {
            "headless": True,
            "args": [
                '--no-sandbox',
                '--disable-setuid-sandbox',
                '--disable-dev-shm-usage',
                '--disable-gpu',
                '--no-zygote',
                '--single-process'
            ]
        }
        if self.use_proxy and self.proxy:
            opts["proxy"] = {"server": self.proxy}
        self.browser = await self.playwright.chromium.launch(**opts)
        self.context = await self.browser.new_context(user_agent=get_random_user_agent())
        self.page = await self.context.new_page()
        await self.page.set_extra_http_headers({
            'Accept-Language': 'en-US,en;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'Referer': 'https://www.bing.com/'
        })
        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb):
        if self.browser:
            await self.browser.close()
        if self.playwright:
            await self.playwright.stop()

    async def search_bing(self):
        urls = []
        try:
            search_url = f"https://www.bing.com/search?q={self.query}"
            await self.page.goto(search_url, timeout=30000)
            await self.page.wait_for_load_state('networkidle')
            links = await self.page.query_selector_all("li.b_algo h2 a")
            for link in links[:self.num_results]:
                href = await link.get_attribute('href')
                if href:
                    urls.append(href)
            return urls
        except Exception as e:
            logger.error(f"Error searching Bing: {e}")
            return []

    async def get_file_size(self, url):
        try:
            async with self.context.new_page() as page:
                response = await page.request.head(url, timeout=15000)
                length = response.headers.get('Content-Length', None)
                if length:
                    return sizeof_fmt(int(length))
                else:
                    return "Unknown Size"
        except Exception:
            return "Unknown Size"

    async def get_pdf_metadata(self, url):
        try:
            async with self.context.new_page() as page:
                resp = await page.request.get(url, timeout=15000)
                if resp.ok:
                    content = await resp.body()
                    pdf = BytesIO(content)
                    reader = PdfReader(pdf)
                    return {
                        'Title': reader.metadata.get('/Title', 'N/A') if reader.metadata else 'N/A',
                        'Author': reader.metadata.get('/Author', 'N/A') if reader.metadata else 'N/A',
                        'Pages': len(reader.pages),
                    }
                else:
                    return {}
        except Exception:
            return {}

    async def extract_real_download_url(self, url):
        try:
            async with self.context.new_page() as page:
                response = await page.goto(url, wait_until='networkidle', timeout=30000)
                if response and response.headers.get('location'):
                    return response.headers['location']
                return page.url
        except Exception as e:
            logger.error(f"Error extracting real download URL: {e}")
            return url

    async def extract_downloadable_files(self, url, custom_ext_list):
        found_files = []
        try:
            response = await self.page.goto(url, timeout=30000, wait_until='networkidle')
            if not response:
                return []

            final_url = self.page.url
            if '.php' in final_url or 'download' in final_url:
                real_url = await self.extract_real_download_url(final_url)
                if real_url != final_url:
                    found_files.append({
                        'url': real_url,
                        'filename': os.path.basename(urlparse(real_url).path) or 'downloaded_file',
                        'size': await self.get_file_size(real_url),
                        'metadata': {}
                    })
                    return found_files

            await self.page.wait_for_load_state('networkidle', timeout=30000)
            content = await self.page.content()
            soup = BeautifulSoup(content, 'html.parser')
            
            default_exts = ['.pdf', '.docx', '.doc', '.zip', '.rar', '.mp3', '.mp4', 
                            '.avi', '.mkv', '.png', '.jpg', '.jpeg', '.gif', '.xlsx', 
                            '.pptx', '.odt', '.txt']
            all_exts = set(default_exts + [ext.strip().lower() for ext in custom_ext_list if ext.strip()])
            
            parsed_base = urlparse(final_url)
            base_url = f"{parsed_base.scheme}://{parsed_base.netloc}"
            
            for a in soup.find_all('a', href=True):
                href = a['href'].strip()
                
                if '.php' in href.lower() or 'download' in href.lower():
                    full_url = href if href.startswith('http') else f"{base_url}{href}"
                    real_url = await self.extract_real_download_url(full_url)
                    if real_url and real_url != full_url:
                        found_files.append({
                            'url': real_url,
                            'filename': os.path.basename(urlparse(real_url).path) or 'downloaded_file',
                            'size': await self.get_file_size(real_url),
                            'metadata': {}
                        })
                        continue

                if any(href.lower().endswith(ext) for ext in all_exts):
                    file_url = href if href.startswith('http') else f"{base_url}{href}"
                    size_str = await self.get_file_size(file_url)
                    meta = {}
                    if file_url.lower().endswith('.pdf'):
                        meta = await self.get_pdf_metadata(file_url)
                    found_files.append({
                        'url': file_url,
                        'filename': os.path.basename(file_url.split('?')[0]),
                        'size': size_str,
                        'metadata': meta
                    })

                # Handle Google Drive links
                elif ("drive.google.com" in href) or ("docs.google.com" in href):
                    file_id = None
                    for pattern in [r'/file/d/([^/]+)', r'id=([^&]+)', r'open\?id=([^&]+)']:
                        match = re.search(pattern, href)
                        if match:
                            file_id = match.group(1)
                            break
                    if file_id:
                        direct_url = f"https://drive.google.com/uc?export=download&id={file_id}"
                        filename = file_id
                        try:
                            response = await self.page.request.head(direct_url, timeout=15000)
                            cd = response.headers.get("Content-Disposition", "")
                            if cd:
                                mt = re.search(r'filename\*?="?([^";]+)', cd)
                                if mt:
                                    filename = mt.group(1).strip('"').strip()
                            found_files.append({
                                'url': direct_url,
                                'filename': filename,
                                'size': await self.get_file_size(direct_url),
                                'metadata': {}
                            })
                        except Exception as e:
                            logger.error(f"Error processing Google Drive link: {e}")
            
            seen_urls = set()
            unique_files = []
            for f in found_files:
                if f['url'] not in seen_urls:
                    seen_urls.add(f['url'])
                    unique_files.append(f)
            return unique_files
        except Exception as e:
            logger.error(f"Error extracting files from {url}: {e}")
            return []

    async def download_file(self, file_info, save_dir, referer):
        file_url = file_info['url']
        fname = file_info['filename']
        path = os.path.join(save_dir, fname)
        base, ext = os.path.splitext(fname)
        counter = 1
        while os.path.exists(path):
            path = os.path.join(save_dir, f"{base}_{counter}{ext}")
            counter += 1
        os.makedirs(save_dir, exist_ok=True)
        try:
            if "drive.google.com" in file_url or "docs.google.com" in file_url:
                # Use enhanced Google Drive downloader
                success = await self.download_from_google_drive(file_url, path)
                return path if success else None
                
            # Original code for non-Google Drive downloads
            async with self.context.new_page() as page:
                headers = {
                    'Accept': '*/*',
                    'Accept-Encoding': 'gzip, deflate, br',
                    'Referer': referer
                }
                response = await page.request.get(file_url, headers=headers, timeout=30000)
                if response.status == 200:
                    content = await response.body()
                    with open(path, 'wb') as f:
                        f.write(content)
                    return path
                else:
                    logger.error(f"Download failed with status {response.status}: {file_url}")
                    return None
        except Exception as e:
            logger.error(f"Error downloading {file_url}: {e}")
            return None

    async def download_from_google_drive(self, url, save_path):
        """Enhanced method to download from Google Drive with multiple fallback approaches"""
        # Extract the file ID from different URL formats
        file_id = None
        url_patterns = [
            r'drive\.google\.com/file/d/([^/]+)',
            r'drive\.google\.com/open\?id=([^&]+)',
            r'docs\.google\.com/\w+/d/([^/]+)',
            r'id=([^&]+)',
            r'drive\.google\.com/uc\?id=([^&]+)',
        ]
        
        for pattern in url_patterns:
            match = re.search(pattern, url)
            if match:
                file_id = match.group(1)
                break
        
        if not file_id:
            logger.error(f"Could not extract file ID from URL: {url}")
            return False
        
        # Approach 1: Try with gdown first (when it works)
        try:
            import gdown
            output = gdown.download(url, save_path, quiet=False, fuzzy=True)
            if output and os.path.exists(save_path) and os.path.getsize(save_path) > 0:
                logger.info(f"Successfully downloaded with gdown: {url}")
                return True
        except Exception as e:
            logger.warning(f"gdown download failed: {e}")
        
        # Approach 2: Use Playwright session with cookies
        try:
            async with self.context.new_page() as page:
                # Visit the file viewing page to get cookies
                view_url = f"https://drive.google.com/file/d/{file_id}/view"
                await page.goto(view_url, wait_until='networkidle', timeout=60000)
                
                # Check for view-only permissions
                if await page.query_selector('text="the owner has not granted you permission to download this file"'):
                    logger.warning("File has view-only permissions, attempting workaround")
                    
                    # Check if it's a PDF (we can use the JS method)
                    is_pdf = await page.query_selector('embed[type="application/pdf"]') is not None
                    if is_pdf:
                        # Try JavaScript PDF capture approach for PDFs
                        success = await self.download_viewonly_pdf_with_js(page, save_path)
                        if success:
                            return True
                    
                    # Try direct download attempt for view-only files
                    cookies = await page.context.cookies()
                    cookie_str = "; ".join([f"{c['name']}={c['value']}" for c in cookies])
                    
                    # Try download URL with custom headers and cookies
                    download_url = f"https://drive.google.com/uc?id={file_id}&export=download&confirm=t"
                    await page.goto(download_url, wait_until='networkidle', timeout=60000)
                    
                    headers = {
                        'User-Agent': get_random_user_agent(),
                        'Cookie': cookie_str,
                        'Accept': '*/*',
                    }
                    
                    response = await page.request.get(download_url, headers=headers)
                    if response.status == 200:
                        content = await response.body()
                        with open(save_path, 'wb') as f:
                            f.write(content)
                        return True
                
                # Standard download flow for files with download permission
                download_url = f"https://drive.google.com/uc?export=download&id={file_id}"
                await page.goto(download_url, wait_until='networkidle', timeout=60000)
                
                # Handle large files with confirmation
                confirm_form = await page.query_selector('form#download-form')
                if confirm_form:
                    await confirm_form.evaluate('form => form.submit()')
                    await page.wait_for_load_state('networkidle')
                    
                    # Get cookies after confirmation
                    cookies = await page.context.cookies()
                    cookie_str = "; ".join([f"{c['name']}={c['value']}" for c in cookies])
                    
                    # Get final download URL with confirmation token
                    download_url = f"https://drive.google.com/uc?export=download&id={file_id}&confirm=t"
                    
                    response = await page.request.get(download_url, headers={'Cookie': cookie_str})
                    if response.status == 200:
                        content = await response.body()
                        with open(save_path, 'wb') as f:
                            f.write(content)
                        return True
        except Exception as e:
            logger.warning(f"Playwright download approach failed: {e}")
        
        # Approach 3: Try with requests and session cookies
        try:
            import requests
            
            session = requests.Session()
            session.headers.update({'User-Agent': get_random_user_agent()})
            
            # Get the initial page to obtain cookies
            url = f"https://drive.google.com/uc?id={file_id}&export=download"
            response = session.get(url, stream=True, timeout=30)
            
            # Check for the download confirmation
            confirmation_token = None
            for k, v in response.cookies.items():
                if k.startswith('download_warning'):
                    confirmation_token = v
                    break
            
            # Use the confirmation token if found
            if confirmation_token:
                url = f"https://drive.google.com/uc?id={file_id}&export=download&confirm={confirmation_token}"
            
            # Download the file
            response = session.get(url, stream=True, timeout=60)
            with open(save_path, 'wb') as f:
                for chunk in response.iter_content(chunk_size=1024*1024):
                    if chunk:
                        f.write(chunk)
            
            if os.path.exists(save_path) and os.path.getsize(save_path) > 0:
                return True
        except Exception as e:
            logger.warning(f"Requests session download failed: {e}")
        
        # All approaches failed
        logger.error(f"All download attempts failed for: {url}")
        return False

    async def download_viewonly_pdf_with_js(self, page, save_path):
        """Use JavaScript approach to download view-only PDFs from Google Drive"""
        try:
            logger.info("Attempting to download view-only PDF using JavaScript method")
            
            # Scroll to ensure all pages are loaded
            await page.evaluate("""
                async function scrollToBottom() {
                    const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
                    const container = document.querySelector('.drive-viewer-paginated-scrollable');
                    if (!container) return;
                    
                    const scrollHeight = container.scrollHeight;
                    const viewportHeight = container.clientHeight;
                    const scrollStep = viewportHeight / 2;
                    
                    for (let scrollPos = 0; scrollPos < scrollHeight; scrollPos += scrollStep) {
                        container.scrollTo(0, scrollPos);
                        await delay(500);
                    }
                    
                    // Final scroll to ensure we reached the bottom
                    container.scrollTo(0, scrollHeight);
                    await delay(1000);
                }
                
                return scrollToBottom();
            """)
            
            # Wait for a moment to ensure all images are loaded
            await page.wait_for_timeout(3000)
            
            # Inject the jsPDF library
            await page.evaluate("""
                return new Promise((resolve, reject) => {
                    const script = document.createElement('script');
                    script.src = 'https://cdnjs.cloudflare.com/ajax/libs/jspdf/2.5.1/jspdf.umd.min.js';
                    script.onload = () => resolve(true);
                    script.onerror = () => reject(new Error('Failed to load jsPDF'));
                    document.head.appendChild(script);
                });
            """)
            
            # Wait for the library to load
            await page.wait_for_timeout(1000)
            
            # Execute the PDF creation script
            pdf_data = await page.evaluate("""
                return new Promise(async (resolve) => {
                    // Make sure jsPDF is loaded
                    if (typeof window.jspdf === 'undefined') {
                        window.jspdf = window.jspdf || {};
                    }
                    
                    // Use the jsPDF library
                    const { jsPDF } = window.jspdf;
                    const pdf = new jsPDF();
                    
                    const images = Array.from(document.querySelectorAll('img')).filter(img => 
                        img.src.startsWith('blob:') && img.width > 100 && img.height > 100
                    );
                    
                    if (images.length === 0) {
                        resolve(null);
                        return;
                    }
                    
                    for (let i = 0; i < images.length; i++) {
                        const img = images[i];
                        
                        // Create canvas and draw image
                        const canvas = document.createElement('canvas');
                        canvas.width = img.width;
                        canvas.height = img.height;
                        const ctx = canvas.getContext('2d');
                        ctx.drawImage(img, 0, 0, img.width, img.height);
                        
                        // Add image to PDF
                        const imgData = canvas.toDataURL('image/jpeg', 1.0);
                        
                        // Add a new page for each image except the first one
                        if (i > 0) {
                            pdf.addPage();
                        }
                        
                        // Calculate dimensions to fit page
                        const pageWidth = pdf.internal.pageSize.getWidth();
                        const pageHeight = pdf.internal.pageSize.getHeight();
                        const imgRatio = img.height / img.width;
                        
                        let imgWidth = pageWidth;
                        let imgHeight = imgWidth * imgRatio;
                        
                        // If height exceeds page, scale down
                        if (imgHeight > pageHeight) {
                            imgHeight = pageHeight;
                            imgWidth = imgHeight / imgRatio;
                        }
                        
                        // Center image on page
                        const x = (pageWidth - imgWidth) / 2;
                        const y = (pageHeight - imgHeight) / 2;
                        
                        pdf.addImage(imgData, 'JPEG', x, y, imgWidth, imgHeight);
                    }
                    
                    // Get the PDF as base64
                    const pdfBase64 = pdf.output('datauristring');
                    resolve(pdfBase64);
                });
            """)
            
            if not pdf_data or not pdf_data.startswith('data:application/pdf;base64,'):
                logger.warning("Failed to generate PDF with JavaScript method")
                return False
            
            # Extract the base64 data and save to file
            base64_data = pdf_data.replace('data:application/pdf;base64,', '')
            pdf_bytes = base64.b64decode(base64_data)
            
            with open(save_path, 'wb') as f:
                f.write(pdf_bytes)
            
            if os.path.exists(save_path) and os.path.getsize(save_path) > 0:
                logger.info("Successfully downloaded view-only PDF using JavaScript method")
                return True
            else:
                return False
        
        except Exception as e:
            logger.error(f"Error in JavaScript PDF download method: {e}")
            return False

    async def deep_search(self, url, custom_ext_list=None, sublink_limit=10000, timeout=60):
        if not custom_ext_list:
            custom_ext_list = []
        progress_text = st.empty()
        progress_bar = st.progress(0)
        file_count_text = st.empty()
        try:
            progress_text.text("Analyzing main page...")
            main_files = await self.extract_downloadable_files(url, custom_ext_list)
            initial_count = len(main_files)
            file_count_text.text(f"Found {initial_count} files on main page")
            progress_text.text("Getting sublinks...")
            sublinks = await self.get_sublinks(url, sublink_limit)
            total_links = len(sublinks)
            progress_text.text(f"Found {total_links} sublinks to process")
            if not sublinks:
                progress_bar.progress(1.0)
                return main_files
            all_files = main_files
            for i, sublink in enumerate(sublinks, 1):
                progress = i / total_links
                progress_text.text(f"Processing sublink {i}/{total_links}: {sublink}")
                progress_bar.progress(progress)
                sub_files = await self.extract_downloadable_files(sublink, custom_ext_list)
                all_files.extend(sub_files)
                file_count_text.text(f"Found {len(all_files)} total files")
            seen_urls = set()
            unique_files = []
            for f in all_files:
                if f['url'] not in seen_urls:
                    seen_urls.add(f['url'])
                    unique_files.append(f)
            final_count = len(unique_files)
            progress_text.text(f"Deep search complete!")
            file_count_text.text(f"Found {final_count} unique files")
            progress_bar.progress(1.0)
            return unique_files
        except Exception as e:
            logger.error(f"Deep search error: {e}")
            progress_text.text(f"Error during deep search: {str(e)}")
            return []
        finally:
            await asyncio.sleep(2)
            if not st.session_state.get('keep_progress', False):
                progress_text.empty()
                progress_bar.empty()

    async def get_sublinks(self, url, limit=10000):
        try:
            await self.page.goto(url, timeout=30000)
            content = await self.page.content()
            soup = BeautifulSoup(content, 'html.parser')
            parsed_base = urlparse(url)
            base_url = f"{parsed_base.scheme}://{parsed_base.netloc}"
            links = set()
            for a in soup.find_all('a', href=True):
                href = a['href'].strip()
                if href.startswith('http'):
                    links.add(href)
                elif href.startswith('/'):
                    links.add(f"{base_url}{href}")
            return list(links)[:limit]
        except Exception as e:
            logger.error(f"Error getting sublinks: {e}")
            return []

# Utility Functions for New Features
def extract_keywords(text, n=5):
    doc = nlp_model(text)
    keywords = [token.text for token in doc if token.is_alpha and not token.is_stop][:n]
    return keywords

def analyze_sentiment(text):
    sentiment_analyzer = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english")
    result = sentiment_analyzer(text[:512])[0]
    return result['label'], result['score']

def get_file_hash(file_path):
    hasher = hashlib.md5()
    with open(file_path, 'rb') as f:
        hasher.update(f.read())
    return hasher.hexdigest()

# Main Function
def main():
    if 'initialized' not in st.session_state:
        st.session_state.initialized = True
        st.session_state.discovered_files = []
        st.session_state.current_url = None
        st.session_state.google_creds = None
        st.session_state.selected_files = []
        st.session_state.do_deep_search = False
        st.session_state.deep_search_url = None
        st.session_state.search_results = []

    st.title("Advanced File Downloader")

    with st.sidebar:
        mode = st.radio("Select Mode", ["Manual URL", "Bing Search", "PDF Summarizer"], key="mode_select")
        with st.expander("Advanced Options", expanded=True):
            custom_extensions = st.text_input("Custom File Extensions", placeholder=".csv, .txt, .epub", key="custom_ext_input", help="Enter extensions like .csv, .txt")
            max_sublinks = st.number_input("Maximum Sublinks to Process", min_value=1, max_value=100000, value=10000, step=50, key="max_sublinks_input", help="Max sublinks to scan from main page")
            sublink_timeout = st.number_input("Search Timeout (seconds per sublink)", min_value=1, max_value=3000, value=30, step=5, key="timeout_input", help="Timeout for each sublink")
            use_proxy = st.checkbox("Use Proxy", key="proxy_checkbox")
            proxy = st.text_input("Proxy URL", placeholder="http://proxy:port", key="proxy_input")
        with st.expander("Google Drive Integration", expanded=False):
            if st.button("Start Google Sign-In", key="google_signin_btn"):
                auth_url = get_google_auth_url()
                st.markdown(f"[Click here to authorize]({auth_url})")
            auth_code = st.text_input("Enter authorization code", key="auth_code_input")
            if st.button("Complete Sign-In", key="complete_signin_btn") and auth_code:
                creds, msg = exchange_code_for_credentials(auth_code)
                st.session_state.google_creds = creds
                st.write(msg)

    if mode == "Manual URL":
        st.header("Manual URL Mode")
        url = st.text_input("Enter URL", placeholder="https://example.com", key="url_input")
        col1, col2 = st.columns([3, 1])
        with col1:
            if st.button("Deep Search", use_container_width=True, key="deep_search_btn"):
                if url:
                    custom_ext_list = [ext.strip().lower() for ext in custom_extensions.split(',') if ext.strip()]
                    valid_ext_list = [ext for ext in custom_ext_list if re.match(r'^\.[a-zA-Z0-9]+$', ext)]
                    if custom_ext_list != valid_ext_list:
                        st.warning("Invalid extensions ignored. Use format like '.csv'.")
                    async def run_deep_search():
                        async with DownloadManager(use_proxy=use_proxy, proxy=proxy) as dm:
                            files = await dm.deep_search(url, valid_ext_list, max_sublinks, sublink_timeout)
                            return files
                    files = asyncio.run(run_deep_search())
                    if files:
                        st.session_state.discovered_files = files
                        st.session_state.current_url = url
                        st.success(f"Found {len(files)} files!")
                    else:
                        st.warning("No files found.")

        if st.session_state.discovered_files:
            files = st.session_state.discovered_files
            st.success(f"Found {len(files)} files!")
            col1, col2 = st.columns([1, 4])
            with col1:
                if st.button("Select All", key="select_all_btn"):
                    st.session_state.selected_files = list(range(len(files)))
                if st.button("Clear Selection", key="clear_selection_btn"):
                    st.session_state.selected_files = []
            selected_files = st.multiselect("Select files to download", options=list(range(len(files))), default=st.session_state.selected_files, format_func=lambda x: f"{files[x]['filename']} ({files[x]['size']})", key="file_multiselect")
            st.session_state.selected_files = selected_files
            if selected_files:
                col1, col2, col3, col4 = st.columns(4)
                with col1:
                    download_dir = st.text_input("Download Directory", value="./downloads", key="download_dir_input")
                with col2:
                    create_zip = st.checkbox("Create ZIP file", value=True, key="create_zip_checkbox")
                with col3:
                    delete_after = st.checkbox("Delete after creating ZIP", key="delete_after_checkbox")
                with col4:
                    upload_to_drive = st.checkbox("Upload to Google Drive", key="upload_drive_checkbox")
                if st.button("Download Selected", key="download_btn"):
                    if not os.path.exists(download_dir):
                        os.makedirs(download_dir)
                    async def download_files():
                        downloaded_paths = []
                        progress_bar = st.progress(0)
                        status_text = st.empty()
                        async with DownloadManager(use_proxy=use_proxy, proxy=proxy) as dm:
                            for i, idx in enumerate(selected_files):
                                progress = (i + 1) / len(selected_files)
                                file_info = files[idx]
                                status_text.text(f"Downloading {file_info['filename']}... ({i+1}/{len(selected_files)})")
                                progress_bar.progress(progress)
                                path = await dm.download_file(file_info, download_dir, url)
                                if path:
                                    downloaded_paths.append(path)
                            status_text.empty()
                            progress_bar.empty()
                            return downloaded_paths
                    downloaded = asyncio.run(download_files())
                    if downloaded:
                        st.success(f"Successfully downloaded {len(downloaded)} files")
                        if create_zip:
                            zip_path = create_zip_file(downloaded, download_dir)
                            st.success(f"Created ZIP file: {zip_path}")
                            with open(zip_path, "rb") as f:
                                zip_data = f.read()
                            st.download_button("Download ZIP", data=zip_data, file_name=os.path.basename(zip_path), mime="application/zip")
                            if upload_to_drive and st.session_state.google_creds:
                                drive_service = googleapiclient.discovery.build("drive", "v3", credentials=st.session_state.google_creds)
                                folder_id = create_drive_folder(drive_service, f"Downloads_{urlparse(url).netloc}")
                                drive_id = google_drive_upload(zip_path, st.session_state.google_creds, folder_id)
                                if not isinstance(drive_id, str) or not drive_id.startswith("Error"):
                                    st.success(f"Uploaded to Google Drive. File ID: {drive_id}")
                                else:
                                    st.error(drive_id)
                            if delete_after:
                                for path in downloaded:
                                    try:
                                        os.remove(path)
                                    except Exception as e:
                                        st.warning(f"Could not delete {path}: {e}")
                                st.info("Deleted original files after ZIP creation")
                        else:
                            for path in downloaded:
                                with open(path, "rb") as f:
                                    file_data = f.read()
                                st.download_button(f"Download {os.path.basename(path)}", data=file_data, file_name=os.path.basename(path))

    elif mode == "Bing Search":
        st.header("Bing Search Mode")
        query = st.text_input("Enter search query", key="search_query_input")
        num_results = st.slider("Number of results", 1, 50, 5, key="num_results_slider")
        if st.button("Search", key="search_btn"):
            if query:
                async def run_search():
                    async with DownloadManager(use_proxy=use_proxy, proxy=proxy, query=query, num_results=num_results) as dm:
                        with st.spinner("Searching..."):
                            urls = await dm.search_bing()
                            if urls:
                                st.session_state.search_results = urls
                                st.success(f"Found {len(urls)} results!")
                                for i, url in enumerate(urls, 1):
                                    with st.expander(f"Result {i}: {url}", expanded=(i == 1)):
                                        if st.button(f"Deep Search Result {i}", key=f"deep_search_result_{i}"):
                                            st.session_state.deep_search_url = url
                                            st.session_state.do_deep_search = True
                            else:
                                st.warning("No search results found.")
                asyncio.run(run_search())

    else:  # PDF Summarizer mode
        if summarizer is None:
            st.error("PDF summarization is not available due to model loading errors.")
        else:
            st.header("PDF Summarizer")
            pdf_url = st.text_input("Enter PDF URL", key="pdf_url_input")
            if st.button("Summarize", key="summarize_btn"):
                if pdf_url:
                    with st.spinner("Generating summary..."):
                        try:
                            response = requests.get(pdf_url, stream=True)
                            temp_pdf = tempfile.NamedTemporaryFile(delete=False, suffix=".pdf")
                            with open(temp_pdf.name, "wb") as f:
                                f.write(response.content)
                            reader = PdfReader(temp_pdf.name)
                            text = " ".join([page.extract_text() or "" for page in reader.pages])
                            os.remove(temp_pdf.name)
                            summary = summarizer(text[:3000], max_length=200, min_length=50, do_sample=False)
                            st.write("Summary:", summary[0]['summary_text'])
                        except Exception as e:
                            st.error(f"Error summarizing PDF: {e}")

if __name__ == "__main__":
    main()