Create app/main.py

app/main.py

@@ -0,0 +1,633 @@
+import os
+import asyncio
+import streamlit as st
+from app.ui import (
+    setup_ui, create_sidebar, display_file_results,
+    handle_downloads, handle_google_drive_upload
+)
+from app.download_manager import DownloadManager
+from app.rag_search import EnhancedRAGSearch
+from app.utils import USER_AGENTS
+
+def initialize_session_state():
+    """Initialize session state variables"""
+    if 'files' not in st.session_state:
+        st.session_state.files = []
+    if 'downloaded_paths' not in st.session_state:
+        st.session_state.downloaded_paths = []
+    if 'download_complete' not in st.session_state:
+        st.session_state.download_complete = False
+    if 'selected_tab' not in st.session_state:
+        st.session_state.selected_tab = 0
+    if 'rag_search' not in st.session_state:
+        st.session_state.rag_search = EnhancedRAGSearch()
+    if 'keep_progress' not in st.session_state:
+        st.session_state.keep_progress = False
+    if 'google_credentials' not in st.session_state:
+        st.session_state.google_credentials = None
+    if 'mode' not in st.session_state:
+        st.session_state.mode = "Standard"
+    if 'use_proxy' not in st.session_state:
+        st.session_state.use_proxy = False
+    if 'proxy_string' not in st.session_state:
+        st.session_state.proxy_string = None
+    if 'stealth_mode' not in st.session_state:
+        st.session_state.stealth_mode = True
+
+def main():
+    # Initialize session state
+    initialize_session_state()
+    
+    # Set up UI styling
+    setup_ui()
+    
+    # Create sidebar
+    create_sidebar()
+
+    # Header section
+    col1, col2 = st.columns([5, 1])
+    with col1:
+        st.markdown("<h1 class='main-header'>Advanced File Downloader</h1>", unsafe_allow_html=True)
+    with col2:
+        st.image("https://img.icons8.com/color/96/000000/download--v1.png", width=70)
+    
+    mode_descriptions = {
+        "Standard": "A versatile tool for discovering and downloading files from any website.",
+        "Education Mode": "Optimized for educational resources, exams, and academic materials.",
+        "Research Mode": "Focused on research papers, datasets, and academic publications.",
+        "Media Mode": "Enhanced for finding and downloading images, videos, and audio files."
+    }
+    
+    st.markdown(f"<p class='info-text'>{mode_descriptions[st.session_state.mode]}</p>", unsafe_allow_html=True)
+    
+    # Main tabs
+    tabs = st.tabs(["Search & Download", "Local File Search", "Advanced Configuration", "Help"])
+    
+    # Tab 1: Search & Download
+    with tabs[0]:
+        st.markdown("<h2 class='section-subheader'>Find and Download Files</h2>", unsafe_allow_html=True)
+        
+        col1, col2 = st.columns([3, 1])
+        with col1:
+            url = st.text_input("Enter a URL to search for downloadable files:", 
+                                placeholder="e.g., https://example.com/resources",
+                                value=st.session_state.get('preset_url', ''))
+        with col2:
+            # Initialize search_method with either session state or default value
+            initial_search_method = st.session_state.get('search_method', "Deep Search")
+            search_method = st.selectbox("Search Method", 
+                                         ["Deep Search", "Quick Search", "Exam Site Mode"],
+                                         index=["Deep Search", "Quick Search", "Exam Site Mode"].index(initial_search_method))
+            # Update session state when changed
+            if search_method != st.session_state.get('search_method'):
+                st.session_state.search_method = search_method
+        
+        # Advanced options in an expander
+        with st.expander("Search Options", expanded=False):
+            col1, col2, col3 = st.columns(3)
+            with col1:
+                depth = st.slider("Search Depth", min_value=1, max_value=5, value=2, 
+                                help="Higher values will search more links but take longer")
+                prioritize_pdfs = st.checkbox("Prioritize PDFs", 
+                                            value=st.session_state.get('prioritize_pdfs', True), 
+                                            help="Focus on finding PDF files first")
+            with col2:
+                timeout = st.slider("Timeout (seconds)", min_value=10, max_value=120, value=60)
+                follow_subdomains = st.checkbox("Follow Subdomains", value=True, 
+                                              help="Include links from subdomains in the search")
+            with col3:
+                # Default extensions based on mode
+                default_extensions = {
+                    "Standard": ".pdf,.doc,.docx,.ppt,.pptx,.xls,.xlsx,.zip",
+                    "Education Mode": ".pdf,.doc,.docx,.ppt,.pptx",
+                    "Research Mode": ".pdf,.txt,.csv,.json,.xlsx",
+                    "Media Mode": ".jpg,.png,.mp3,.mp4,.avi,.mov"
+                }
+                
+                custom_extensions = st.text_area(
+                    "Custom File Extensions", 
+                    value=st.session_state.get('custom_extensions', default_extensions[st.session_state.mode]),
+                    help="Comma-separated list of file extensions to look for"
+                )
+                
+                # Update session state when extensions changed
+                if 'custom_extensions' not in st.session_state or custom_extensions != st.session_state.custom_extensions:
+                    st.session_state.custom_extensions = custom_extensions
+                
+        search_col1, search_col2 = st.columns([4, 1])
+        with search_col1:
+            search_button = st.button("🔍 Start Search", use_container_width=True)
+        with search_col2:
+            clear_button = st.button("🧹 Clear Results", use_container_width=True)
+            
+        # File results section
+        if st.session_state.files:
+            # Display file results
+            selected_files, displayed_files = display_file_results(st.session_state.files)
+            
+            # Download options
+            if selected_files:
+                col1, col2 = st.columns(2)
+                with col1:
+                    download_dir = st.text_input("Download Directory", value="downloads")
+                with col2:
+                    download_option = st.radio("Download as", ["Individual Files", "ZIP Archive"], horizontal=True)
+                
+                download_col1, download_col2, download_col3 = st.columns([3, 1, 1])
+                with download_col1:
+                    download_button = st.button("⬇️ Download Selected Files", use_container_width=True)
+                with download_col2:
+                    google_drive_button = st.button("📤 Upload to Drive", 
+                                                  use_container_width=True, 
+                                                  disabled=not st.session_state.google_credentials)
+                with download_col3:
+                    select_all = st.button("Select All Files", use_container_width=True)
+                    
+                # Handle select all button
+                if select_all:
+                    for i in displayed_files:
+                        st.session_state[f"select_{i}"] = True
+                    st.rerun()
+                
+                # Handle download button if clicked
+                if download_button:
+                    # Create download directory
+                    os.makedirs(download_dir, exist_ok=True)
+                    handle_downloads(selected_files, download_dir, download_option, download_col1)
+                
+                # Handle Google Drive upload
+                if google_drive_button:
+                    handle_google_drive_upload(selected_files)
+    
+    # Tab 2: Local File Search
+    with tabs[1]:
+        st.markdown("<h2 class='section-subheader'>Search Downloaded Files</h2>", unsafe_allow_html=True)
+        st.write("Upload files to search through their content with AI-powered semantic search.")
+        
+        # File upload
+        uploaded_files = st.file_uploader("Upload documents for search", 
+                                         accept_multiple_files=True, 
+                                         type=['pdf', 'docx', 'txt', 'csv', 'json'])
+        
+        if uploaded_files:
+            # Build search index on upload
+            col1, col2 = st.columns([4, 1])
+            with col1:
+                use_transformer = st.checkbox("Use AI Transformer Model", value=st.session_state.rag_search.use_transformer, 
+                                           help="Uses advanced AI for more accurate semantic search (if available)")
+            with col2:
+                if st.button("Build Search Index", use_container_width=True):
+                    with st.spinner("Processing files and building search index..."):
+                        files_added = 0
+                        for uploaded_file in uploaded_files:
+                            file_info = {
+                                'filename': uploaded_file.name,
+                                'url': f'local://{uploaded_file.name}',
+                                'size': humanize_file_size(uploaded_file.size)
+                            }
+                            success = st.session_state.rag_search.add_file(uploaded_file.getvalue(), file_info)
+                            if success:
+                                files_added += 1
+                        
+                        if files_added > 0:
+                            index_built = st.session_state.rag_search.build_index()
+                            if index_built:
+                                st.success(f"✅ Successfully indexed {files_added} files!")
+                            else:
+                                st.error("Failed to build search index.")
+                        else:
+                            st.warning("No valid text could be extracted from the files.")
+            
+            # Search interface
+            st.markdown("<h3 class='section-subheader'>Search Files</h3>", unsafe_allow_html=True)
+            
+            col1, col2 = st.columns([4, 1])
+            with col1:
+                query = st.text_input("Enter search query:", placeholder="e.g., neural networks, climate change")
+            with col2:
+                expand_query = st.checkbox("Auto-expand query", value=True, 
+                                         help="Automatically add related terms to your search")
+            
+            col1, col2 = st.columns([4, 1])
+            with col1:
+                if st.button("🔍 Search Documents", use_container_width=True):
+                    if not query:
+                        st.warning("Please enter a search query")
+                    else:
+                        with st.spinner("Searching..."):
+                            results = st.session_state.rag_search.search(query, top_k=5, search_chunks=True)
+                            
+                            if results:
+                                st.markdown(f"**Found {len(results)} relevant documents:**")
+                                for i, result in enumerate(results):
+                                    with st.container():
+                                        st.markdown(f"<div class='result-card'>", unsafe_allow_html=True)
+                                        st.markdown(f"**{i+1}. {result['file_info']['filename']}** (Score: {result['score']:.2f})")
+                                        
+                                        if result.get('chunk_preview'):
+                                            st.markdown("**Matching content:**")
+                                            st.text(result['chunk_preview'])
+                                        
+                                        st.markdown("</div>", unsafe_allow_html=True)
+                            else:
+                                st.info("No matching documents found. Try a different query.")
+            with col2:
+                num_results = st.number_input("Max results", min_value=1, max_value=20, value=5)
+            
+            # Quick search tips
+            with st.expander("Search Tips", expanded=False):
+                st.markdown("""
+                ### Effective Search Tips
+                
+                - **Be specific** with your queries for more accurate results
+                - **Try different phrasings** if you don't get the results you expect
+                - Use **quotation marks** for exact phrase matching
+                - For **complex topics**, break down your search into multiple queries
+                - **Combine related terms** to improve recall
+                
+                The search engine uses advanced algorithms to understand the semantic meaning of your query, 
+                not just keyword matching.
+                """)
+    
+    # Tab 3: Advanced Configuration
+    with tabs[2]:
+        st.markdown("<h2 class='section-subheader'>Advanced Settings</h2>", unsafe_allow_html=True)
+        
+        config_tabs = st.tabs(["Browser Settings", "Proxy Configuration", "Download Options", "System"])
+        
+        # Browser Settings tab
+        with config_tabs[0]:
+            col1, col2 = st.columns(2)
+            with col1:
+                use_stealth = st.checkbox("Use Stealth Mode", value=st.session_state.stealth_mode, 
+                                        help="Makes browser harder to detect as automated, but may be slower")
+                
+                handle_captchas = st.checkbox("Handle Captchas Automatically", value=False, 
+                                           help="Attempt to solve simple captchas automatically")
+                
+                download_timeout = st.slider("Download Timeout (seconds)", 
+                                          min_value=30, max_value=600, value=300,
+                                          help="Maximum time to wait for downloads to complete")
+            with col2:
+                user_agent = st.selectbox("User Agent", USER_AGENTS, index=0,
+                                        help="Browser identity to use when accessing websites")
+                
+                save_screenshots = st.checkbox("Save Browser Screenshots", value=False,
+                                           help="Save screenshots when errors occur for debugging")
+                
+                browser_lang = st.selectbox("Browser Language", 
+                                         ["English (US)", "English (UK)", "Spanish", "French", "German", "Chinese"],
+                                         index=0)
+            
+            if st.button("Update Browser Settings"):
+                st.session_state.stealth_mode = use_stealth
+                st.success("Browser settings updated!")
+                
+            # Dependency installation section
+            st.markdown("<h4 class='section-subheader'>Dependencies</h4>", unsafe_allow_html=True)
+            if st.button("Install Playwright Dependencies"):
+                from app.ui import install_playwright_dependencies
+                with st.spinner("Installing dependencies..."):
+                    install_playwright_dependencies()
+        
+        # Proxy Configuration tab
+        with config_tabs[1]:
+            proxy_enabled = st.checkbox("Enable Proxy", value=st.session_state.use_proxy,
+                                     help="Route requests through a proxy server for anonymity or bypassing restrictions")
+            
+            if proxy_enabled:
+                proxy_col1, proxy_col2 = st.columns(2)
+                with proxy_col1:
+                    proxy_type = st.selectbox("Proxy Type", ["HTTP", "SOCKS5", "HTTPS"])
+                    proxy_host = st.text_input("Proxy Host", placeholder="e.g., 127.0.0.1")
+                with proxy_col2:
+                    proxy_port = st.text_input("Proxy Port", placeholder="e.g., 8080")
+                    proxy_auth = st.text_input("Proxy Authentication (optional)", 
+                                            placeholder="username:password", type="password")
+            
+            st.markdown("<h4 class='section-subheader'>Proxy Rotation</h4>", unsafe_allow_html=True)
+            use_proxy_rotation = st.checkbox("Enable Proxy Rotation", value=False,
+                                          help="Automatically rotate between multiple proxies for better anonymity")
+            
+            if use_proxy_rotation:
+                proxy_list = st.text_area("Proxy List (one per line)", 
+                                       placeholder="http://proxy1.example.com:8080\nhttp://proxy2.example.com:8080")
+                rotation_interval = st.slider("Rotation Interval (requests)", 
+                                           min_value=1, max_value=50, value=10,
+                                           help="How often to switch proxies")
+            
+            if st.button("Save Proxy Configuration"):
+                # Construct the proxy string
+                proxy_string = None
+                if proxy_enabled and proxy_host and proxy_port:
+                    proxy_prefix = f"{proxy_type.lower()}://"
+                    proxy_auth_str = f"{proxy_auth}@" if proxy_auth else ""
+                    proxy_string = f"{proxy_prefix}{proxy_auth_str}{proxy_host}:{proxy_port}"
+                
+                # Update session state
+                st.session_state.use_proxy = proxy_enabled
+                st.session_state.proxy_string = proxy_string
+                
+                # Configure proxy rotation if enabled
+                from app.utils import PROXY_ROTATION_CONFIG
+                if use_proxy_rotation and proxy_list:
+                    PROXY_ROTATION_CONFIG["enabled"] = True
+                    PROXY_ROTATION_CONFIG["rotation_interval"] = rotation_interval
+                    PROXY_ROTATION_CONFIG["proxies"] = [p.strip() for p in proxy_list.splitlines() if p.strip()]
+                
+                st.success("Proxy configuration updated!")
+        
+        # Download Options tab
+        with config_tabs[2]:
+            col1, col2 = st.columns(2)
+            with col1:
+                st.markdown("<h4 class='section-subheader'>Download Behavior</h4>", unsafe_allow_html=True)
+                
+                skip_existing = st.checkbox("Skip Existing Files", value=True,
+                                        help="Don't download files that already exist locally")
+                
+                auto_rename = st.checkbox("Auto-Rename Duplicates", value=True,
+                                       help="Automatically rename files instead of overwriting")
+                
+                verify_downloads = st.checkbox("Verify Downloads", value=True,
+                                           help="Check file integrity after download")
+                
+                max_retries = st.slider("Max Retries", min_value=0, max_value=10, value=3,
+                                     help="Number of times to retry failed downloads")
+            
+            with col2:
+                st.markdown("<h4 class='section-subheader'>File Organization</h4>", unsafe_allow_html=True)
+                
+                auto_organize = st.checkbox("Auto-Organize Files", value=True,
+                                         help="Automatically organize files by type")
+                
+                default_dir = st.text_input("Default Download Directory", value="downloads",
+                                         help="Default location to save downloaded files")
+                
+                org_by_domain = st.checkbox("Organize by Domain", value=False,
+                                        help="Create subdirectories based on source domains")
+                
+                org_by_type = st.checkbox("Organize by File Type", value=False,
+                                       help="Create subdirectories based on file types")
+            
+            if st.button("Save Download Settings"):
+                st.session_state.download_settings = {
+                    "skip_existing": skip_existing,
+                    "auto_rename": auto_rename,
+                    "verify_downloads": verify_downloads,
+                    "max_retries": max_retries,
+                    "auto_organize": auto_organize,
+                    "default_dir": default_dir,
+                    "org_by_domain": org_by_domain,
+                    "org_by_type": org_by_type
+                }
+                st.success("Download settings saved!")
+        
+        # System tab
+        with config_tabs[3]:
+            col1, col2 = st.columns(2)
+            with col1:
+                st.markdown("<h4 class='section-subheader'>Memory & Performance</h4>", unsafe_allow_html=True)
+                
+                max_concurrent = st.slider("Max Concurrent Downloads", min_value=1, max_value=10, value=3,
+                                        help="Maximum number of simultaneous downloads")
+                
+                memory_limit = st.slider("Memory Limit (MB)", min_value=256, max_value=4096, value=1024,
+                                      help="Maximum memory to use for file processing")
+                
+                processing_threads = st.slider("Processing Threads", min_value=1, max_value=8, value=2,
+                                           help="Number of threads to use for file processing")
+            
+            with col2:
+                st.markdown("<h4 class='section-subheader'>Logs & Diagnostics</h4>", unsafe_allow_html=True)
+                
+                log_level = st.selectbox("Log Level", ["DEBUG", "INFO", "WARNING", "ERROR"], index=1,
+                                      help="Detail level for application logs")
+                
+                save_debug_info = st.checkbox("Save Debug Information", value=False,
+                                           help="Save detailed information about program execution")
+                
+                log_dir = st.text_input("Log Directory", value="logs",
+                                     help="Directory to save log files")
+            
+            if st.button("Apply System Settings"):
+                import logging
+                st.session_state.system_settings = {
+                    "max_concurrent": max_concurrent,
+                    "memory_limit": memory_limit,
+                    "processing_threads": processing_threads,
+                    "log_level": log_level,
+                    "save_debug_info": save_debug_info,
+                    "log_dir": log_dir
+                }
+                # Update logging configuration
+                log_level_num = getattr(logging, log_level)
+                logging.getLogger().setLevel(log_level_num)
+                st.success("System settings applied!")
+                
+            # Reset application button
+            st.markdown("<h4 class='section-subheader'>Application Control</h4>", unsafe_allow_html=True)
+            reset_col1, reset_col2 = st.columns([1, 3])
+            with reset_col1:
+                if st.button("Reset Application", use_container_width=True):
+                    for key in list(st.session_state.keys()):
+                        if key != 'google_credentials':  # Preserve Google auth
+                            del st.session_state[key]
+                    st.success("Application has been reset!")
+                    st.rerun()
+            with reset_col2:
+                st.info("This will clear all search results, downloaded files, and reset settings to defaults.")
+    
+    # Tab 4: Help
+    with tabs[3]:
+        st.markdown("<h2 class='section-subheader'>Help & Documentation</h2>", unsafe_allow_html=True)
+        
+        help_tabs = st.tabs(["Quick Start", "Advanced Features", "Troubleshooting", "About"])
+        
+        with help_tabs[0]:
+            st.markdown("""
+            ### Getting Started
+            
+            1. **Enter a URL** on the Search & Download tab
+            2. Select a **Search Method**:
+               - **Deep Search**: Thorough but slower
+               - **Quick Search**: Fast but may miss some files
+               - **Exam Site Mode**: Optimized for educational resource sites
+            3. Click **Start Search** to find downloadable files
+            4. Select files you want to download
+            5. Click **Download Selected Files**
+            
+            #### Using Different Modes
+            
+            Select a mode from the sidebar to optimize the tool for different use cases:
+            
+            - **Standard Mode**: Balanced for general use
+            - **Education Mode**: Optimized for finding academic materials
+            - **Research Mode**: Better for research papers and datasets
+            - **Media Mode**: Enhanced for finding images, videos, and audio
+            
+            For best results with educational materials, use the **Exam Site Mode** with websites that contain past exams, papers, or course materials.
+            """)
+            
+        with help_tabs[1]:
+            st.markdown("""
+            ### Advanced Features
+            
+            - **Local File Search**: Upload files and search through their content using the enhanced RAG search
+            - **Custom Extensions**: Specify additional file types to look for beyond the default set
+            - **Stealth Mode**: Makes the browser harder to detect as automated, useful for sites that block scrapers
+            - **Proxy Support**: Use proxies to access region-restricted content or improve anonymity
+            - **Google Drive Integration**: Upload downloaded files directly to your Google Drive
+            
+            #### Search Tips
+            
+            - For educational sites, include specific terms like "exam", "test", "paper" in the URL
+            - When using Local File Search, try different variations of your query for better results
+            - Use filtering and sorting options to find the most relevant files quickly
+            
+            #### File Organization
+            
+            You can configure automatic file organization in the Advanced Configuration tab:
+            
+            - **Organize by Domain**: Creates folders based on the source website
+            - **Organize by File Type**: Separates files into folders by their extension
+            - **Auto-Rename**: Prevents overwriting existing files with same names
+            """)
+            
+        with help_tabs[2]:
+            st.markdown("""
+            ### Troubleshooting
+            
+            #### Common Issues
+            
+            - **No files found**: Try using Deep Search with higher depth value, or add more specific file extensions
+            - **Downloads failing**: Check if the site requires authentication or uses captchas
+            - **Slow performance**: Reduce search depth or disable stealth mode for faster results
+            - **Browser errors**: Click "Install Playwright Dependencies" in Advanced Settings
+            
+            #### Captcha Issues
+            
+            Some websites use captchas to prevent automated access. If you encounter captchas:
+            
+            1. Try using a different proxy
+            2. Enable "Handle Captchas Automatically" for simple captchas
+            3. For complex captchas, you may need to manually access the site first
+            
+            #### Proxy Problems
+            
+            If you're having issues with proxies:
+            
+            1. Verify your proxy is working with an external tool
+            2. Check that you've entered the correct format (http://host:port)
+            3. Some websites may block known proxy IPs
+            
+            #### Memory Usage
+            
+            If the application is using too much memory:
+            
+            1. Reduce the "Memory Limit" in System settings
+            2. Process fewer files at once
+            3. Use lower search depth values
+            """)
+            
+        with help_tabs[3]:
+            st.markdown("""
+            ### About This Tool
+            
+            **Advanced File Downloader** is a sophisticated tool designed to discover and download files from websites with enhanced capabilities for educational resources.
+            
+            #### Key Features
+            
+            - **Smart Discovery**: Finds downloadable files even when they're not directly linked
+            - **Enhanced RAG Search**: Search through downloaded documents using advanced AI techniques
+            - **Educational Focus**: Specialized detection for exam papers and academic resources
+            - **Stealth Capabilities**: Avoids detection by anti-scraping measures
+            
+            #### Technical Details
+            
+            This tool uses:
+            
+            - **Playwright**: For browser automation and stealth capabilities
+            - **Sentence Transformers**: For AI-powered semantic search
+            - **Streamlit**: For the user interface
+            - **Google Drive API**: For cloud integration
+            
+            #### Credits
+            
+            Created with Python, Streamlit, Playwright, and various AI libraries.
+            
+            For issues or suggestions, please contact the developer.
+            
+            Version 2.0 - March 2025
+            """)
+    
+    # Handle search button
+    if search_button and url:
+        # Reset files and downloaded paths
+        st.session_state.files = []
+        st.session_state.downloaded_paths = []
+        st.session_state.download_complete = False
+        
+        # Clear the preset URL if it was used
+        if 'preset_url' in st.session_state:
+            st.session_state.preset_url = ''
+        
+        # Prepare custom extensions
+        custom_ext_list = [ext.strip() for ext in custom_extensions.split(",") if ext.strip()]
+        
+        # Configure search parameters based on method
+        sublink_limit = 5000 if search_method == "Deep Search" else 1000
+        search_depth = depth if search_method == "Deep Search" else 1
+        is_exam_site = search_method == "Exam Site Mode"
+        
+        # Execute the search asynchronously
+        async def run_search():
+            async with DownloadManager(
+                use_proxy=st.session_state.use_proxy,
+                proxy=st.session_state.proxy_string,
+                use_stealth=st.session_state.stealth_mode
+            ) as manager:
+                # For exam sites, use specialized approach
+                if is_exam_site:
+                    st.session_state.keep_progress = True
+                    edu_links = await manager.get_edu_exam_links(url)
+                    all_files = []
+                    
+                    progress_text = st.empty()
+                    progress_bar = st.progress(0)
+                    
+                    # Process each exam link
+                    for i, link in enumerate(edu_links):
+                        progress = (i+1) / max(1, len(edu_links))
+                        progress_text.text(f"Processing exam link {i+1}/{len(edu_links)}: {link}")
+                        progress_bar.progress(progress)
+                        
+                        files = await manager.extract_downloadable_files(link, custom_ext_list)
+                        all_files.extend(files)
+                    
+                    st.session_state.files = all_files
+                    progress_text.empty()
+                    progress_bar.empty()
+                    st.session_state.keep_progress = False
+                    
+                else:
+                    # Use general search method
+                    files = await manager.deep_search(url, custom_ext_list, sublink_limit, timeout)
+                    st.session_state.files = files
+        
+        # Run the search
+        asyncio.run(run_search())
+        st.rerun()
+    
+    # Handle clear button
+    if clear_button:
+        st.session_state.files = []
+        st.session_state.downloaded_paths = []
+        st.session_state.download_complete = False
+        if 'preset_url' in st.session_state:
+            st.session_state.preset_url = ''
+        st.rerun()
+
+# Entry point
+if __name__ == "__main__":
+    main()