tag links
+ a_links = await self.page.query_selector_all('a[href]')
+ for a in a_links:
+ try:
+ href = await a.get_attribute('href')
+ if href and not href.startswith('javascript:') and not href.startswith('#'):
+ full_url = href if href.startswith('http') else self.resolve_relative_url(href, base_url, path_base)
+ links_set.add(full_url)
+ except Exception:
+ pass
+
+ # Get iframe sources
+ iframes = await self.page.query_selector_all('iframe[src]')
+ for iframe in iframes:
+ try:
+ src = await iframe.get_attribute('src')
+ if src and not src.startswith('javascript:') and not src.startswith('about:'):
+ full_url = src if src.startswith('http') else self.resolve_relative_url(src, base_url, path_base)
+ links_set.add(full_url)
+ except Exception:
+ pass
+
+ # Get links from onclick attributes that reference URLs
+ onclick_elements = await self.page.query_selector_all('*[onclick*="window.location"], *[onclick*="document.location"]')
+ for el in onclick_elements:
+ try:
+ onclick = await el.get_attribute('onclick')
+ urls = re.findall(r'(https?://[^\'"]+)', onclick)
+ for url in urls:
+ links_set.add(url)
+ except Exception:
+ pass
+
+ # Look for URLs in data-* attributes
+ data_elements = await self.page.query_selector_all('*[data-url], *[data-href], *[data-src]')
+ for el in data_elements:
+ for attr in ['data-url', 'data-href', 'data-src']:
+ try:
+ value = await el.get_attribute(attr)
+ if value and not value.startswith('javascript:'):
+ full_url = value if value.startswith('http') else self.resolve_relative_url(value, base_url, path_base)
+ links_set.add(full_url)
+ except Exception:
+ pass
+
+ # Look for special anchor links that might not have href attributes
+ special_anchors = await self.page.query_selector_all('.rgMasterTable a, .grid a, #GridView1 a, #gvResults a')
+ for anchor in special_anchors:
+ try:
+ href = await anchor.get_attribute('href')
+ if href and not href.startswith('javascript:') and not href.startswith('#'):
+ full_url = href if href.startswith('http') else self.resolve_relative_url(href, base_url, path_base)
+ links_set.add(full_url)
+ except Exception:
+ pass
+
+ # Extract links from JSON data embedded in the page
+ script_elements = await self.page.query_selector_all('script[type="application/json"], script[type="text/json"]')
+ for script in script_elements:
+ try:
+ script_content = await script.text_content()
+ if script_content:
+ # Look for URLs in the JSON content
+ urls = re.findall(r'(https?://[^\'"]+)', script_content)
+ for url in urls:
+ links_set.add(url)
+ except Exception:
+ pass
+
+ def resolve_relative_url(self, relative_url, base_url, path_base):
+ """Properly resolve relative URLs considering multiple formats"""
+ if relative_url.startswith('/'):
+ # Absolute path relative to domain
+ return f"{base_url}{relative_url}"
+ elif relative_url.startswith('./'):
+ # Explicit relative path
+ return f"{base_url}{path_base}/{relative_url[2:]}"
+ elif relative_url.startswith('../'):
+ # Parent directory
+ parent_path = '/'.join(path_base.split('/')[:-1])
+ return f"{base_url}{parent_path}/{relative_url[3:]}"
+ else:
+ # Regular relative path
+ return f"{base_url}{path_base}/{relative_url}"
+
+ async def deep_search(self, url, custom_ext_list=None, sublink_limit=10000, timeout=60):
+ if not custom_ext_list:
+ custom_ext_list = []
+ progress_text = st.empty()
+ progress_bar = st.progress(0)
+ file_count_text = st.empty()
+
+ try:
+ # Reset the visited URLs for a fresh deep search
+ self.visited_urls = set()
+
+ progress_text.text("Analyzing main page...")
+ # Special handling for ASP.NET pages
+ is_aspnet = False
+ try:
+ await self.page.goto(url, timeout=30000, wait_until='networkidle')
+ is_aspnet = await self.page.evaluate('''
+ () => {
+ return document.querySelector('form#aspnetForm') !== null ||
+ document.querySelector('input[name="__VIEWSTATE"]') !== null;
+ }
+ ''')
+ except Exception:
+ pass
+
+ # Check if this URL is a direct download
+ if is_download_link(url):
+ progress_text.text("URL appears to be a direct download. Analyzing...")
+
+ # Try to extract file directly
+ normalized_url = normalize_download_url(url)
+ file_info = {
+ 'url': normalized_url,
+ 'download_url': normalized_url,
+ 'filename': os.path.basename(urlparse(normalized_url).path) or 'download',
+ 'size': 'Unknown Size',
+ 'metadata': {}
+ }
+
+ # Add to visited URLs
+ self.visited_urls.add(normalized_url)
+ progress_bar.progress(1.0)
+ return [file_info]
+
+ # Extract files from main page
+ main_files = await self.extract_downloadable_files(url, custom_ext_list)
+ initial_count = len(main_files)
+ file_count_text.text(f"Found {initial_count} files on main page")
+
+ # Get sublinks with enhanced method
+ progress_text.text("Getting sublinks...")
+ sublinks = await self.get_sublinks(url, sublink_limit)
+ total_links = len(sublinks)
+ progress_text.text(f"Found {total_links} sublinks to process")
+
+ # Always include files from the main page, regardless of sublinks
+ all_files = main_files
+
+ if not sublinks:
+ progress_bar.progress(1.0)
+ return all_files
+
+ # Process each sublink
+ for i, sublink in enumerate(sublinks, 1):
+ progress = i / total_links
+ progress_text.text(f"Processing sublink {i}/{total_links}: {sublink}")
+ progress_bar.progress(progress)
+
+ try:
+ # Check if this is a direct download link
+ if is_download_link(sublink):
+ # For download links, just add the link directly
+ normalized_url = normalize_download_url(sublink)
+
+ # Skip if already visited
+ if normalized_url in self.visited_urls:
+ continue
+
+ # Mark as visited
+ self.visited_urls.add(normalized_url)
+
+ # Get file size if possible
+ size_str = await self.get_file_size(normalized_url)
+
+ # Get filename, with fallback to domain-based name
+ filename = os.path.basename(urlparse(normalized_url).path)
+ if not filename or filename == '/' or '?' in filename:
+ domain = get_domain(normalized_url)
+ ext = '.pdf' # Default extension
+ for common_ext in ['.pdf', '.doc', '.docx', '.xls', '.xlsx', '.txt', '.zip']:
+ if common_ext in normalized_url.lower():
+ ext = common_ext
+ break
+ filename = f"file_from_{domain}{ext}"
+
+ # Add file to results
+ all_files.append({
+ 'url': normalized_url,
+ 'download_url': normalized_url,
+ 'filename': filename,
+ 'size': size_str,
+ 'metadata': {}
+ })
+ file_count_text.text(f"Found {len(all_files)} total files")
+ continue
+
+ # For regular links, use a longer timeout for ASP.NET pages which can be slower
+ sub_timeout = timeout * 2 if is_aspnet else timeout
+
+ # Skip already visited URLs
+ if sublink in self.visited_urls:
+ continue
+
+ # Extract files from sublink
+ sub_files = await self.extract_downloadable_files(sublink, custom_ext_list)
+ all_files.extend(sub_files)
+ file_count_text.text(f"Found {len(all_files)} total files")
+ except Exception as e:
+ logger.warning(f"Error processing sublink {sublink}: {e}")
+
+ # Deduplicate files
+ seen_urls = set()
+ unique_files = []
+ for f in all_files:
+ if f['url'] not in seen_urls:
+ seen_urls.add(f['url'])
+ unique_files.append(f)
+
+ final_count = len(unique_files)
+ progress_text.text(f"Deep search complete!")
+ file_count_text.text(f"Found {final_count} unique files")
+ progress_bar.progress(1.0)
+ return unique_files
+
+ except Exception as e:
+ logger.error(f"Deep search error: {e}")
+ progress_text.text(f"Error during deep search: {str(e)}")
+ return []
+
+ finally:
+ await asyncio.sleep(2)
+ if not st.session_state.get('keep_progress', False):
+ progress_text.empty()
+ progress_bar.empty()
+
+# -------------------- Main App --------------------
+def main():
+
+ # Custom CSS for better appearance
+ st.markdown("""
+
+ """, unsafe_allow_html=True)
+
+ # Initialize session state for storing files
if 'files' not in st.session_state:
st.session_state.files = []
if 'downloaded_paths' not in st.session_state:
@@ -66,15 +3887,976 @@ def initialize_session_state():
st.session_state.proxy_string = None
if 'stealth_mode' not in st.session_state:
st.session_state.stealth_mode = True
+
+ # ============================
+ # SIDEBAR
+ # ============================
+ with st.sidebar:
+ st.image("https://img.icons8.com/color/96/000000/download--v1.png", width=50)
+ st.markdown("", unsafe_allow_html=True)
+
+ # Mode Selection
+ st.markdown("", unsafe_allow_html=True)
+
+ # Quick Settings
+ st.markdown("", unsafe_allow_html=True)
+
+ # Google Drive Integration
+ st.markdown("", unsafe_allow_html=True)
+
+ # Preset buttons for common EDU sites
+ if st.session_state.mode == "Education Mode":
+ st.markdown("", unsafe_allow_html=True)
+
+ # Tool status
+ st.markdown("", unsafe_allow_html=True)
+
+ # App info
+ st.markdown("", unsafe_allow_html=True)
-# Import the UI code while keeping the modular structure
-from ui import setup_ui, create_sidebar, display_file_results, handle_downloads, handle_google_drive_upload
-from main import main as app_main
-
-# Set up and run the application
-def main():
- initialize_session_state()
- app_main()
+ # ============================
+ # MAIN CONTENT AREA
+ # ============================
+
+ # Header section
+ col1, col2 = st.columns([5, 1])
+ with col1:
+ st.markdown("Advanced File Downloader
", unsafe_allow_html=True)
+ with col2:
+ st.image("https://img.icons8.com/color/96/000000/download--v1.png", width=70)
+
+ mode_descriptions = {
+ "Standard": "A versatile tool for discovering and downloading files from any website.",
+ "Education Mode": "Optimized for educational resources, exams, and academic materials.",
+ "Research Mode": "Focused on research papers, datasets, and academic publications.",
+ "Media Mode": "Enhanced for finding and downloading images, videos, and audio files."
+ }
+
+ st.markdown(f"{mode_descriptions[st.session_state.mode]}
", unsafe_allow_html=True)
+
+ # Main tabs
+ tabs = st.tabs(["Search & Download", "Local File Search", "Advanced Configuration", "Help"])
+
+ # Tab 1: Search & Download
+ with tabs[0]:
+ st.markdown("", unsafe_allow_html=True)
+
+ col1, col2 = st.columns([3, 1])
+ with col1:
+ url = st.text_input("Enter a URL to search for downloadable files:",
+ placeholder="e.g., https://example.com/resources",
+ value=st.session_state.get('preset_url', ''))
+ with col2:
+ # Initialize search_method with either session state or default value
+ initial_search_method = st.session_state.get('search_method', "Deep Search")
+ search_method = st.selectbox("Search Method",
+ ["Deep Search", "Quick Search", "Exam Site Mode"],
+ index=["Deep Search", "Quick Search", "Exam Site Mode"].index(initial_search_method))
+ # Update session state when changed
+ if search_method != st.session_state.get('search_method'):
+ st.session_state.search_method = search_method
+
+ # Advanced options in an expander
+ with st.expander("Search Options", expanded=False):
+ col1, col2, col3 = st.columns(3)
+ with col1:
+ depth = st.slider("Search Depth", min_value=1, max_value=5, value=2,
+ help="Higher values will search more links but take longer")
+ prioritize_pdfs = st.checkbox("Prioritize PDFs",
+ value=st.session_state.get('prioritize_pdfs', True),
+ help="Focus on finding PDF files first")
+ with col2:
+ timeout = st.slider("Timeout (seconds)", min_value=10, max_value=120, value=60)
+ follow_subdomains = st.checkbox("Follow Subdomains", value=True,
+ help="Include links from subdomains in the search")
+ with col3:
+ # Default extensions based on mode
+ default_extensions = {
+ "Standard": ".pdf,.doc,.docx,.ppt,.pptx,.xls,.xlsx,.zip",
+ "Education Mode": ".pdf,.doc,.docx,.ppt,.pptx",
+ "Research Mode": ".pdf,.txt,.csv,.json,.xlsx",
+ "Media Mode": ".jpg,.png,.mp3,.mp4,.avi,.mov"
+ }
+
+ custom_extensions = st.text_area(
+ "Custom File Extensions",
+ value=st.session_state.get('custom_extensions', default_extensions[st.session_state.mode]),
+ help="Comma-separated list of file extensions to look for"
+ )
+
+ # Update session state when extensions changed
+ if 'custom_extensions' not in st.session_state or custom_extensions != st.session_state.custom_extensions:
+ st.session_state.custom_extensions = custom_extensions
+
+ search_col1, search_col2 = st.columns([4, 1])
+ with search_col1:
+ search_button = st.button("๐ Start Search", use_container_width=True)
+ with search_col2:
+ clear_button = st.button("๐งน Clear Results", use_container_width=True)
+
+ # File results section
+ if st.session_state.files:
+ st.markdown("", unsafe_allow_html=True)
+
+ # File filtering options
+ filter_col1, filter_col2, filter_col3 = st.columns([2, 2, 1])
+ with filter_col1:
+ file_filter = st.text_input("Filter files by name:", placeholder="e.g., exam, 2023, etc.")
+ with filter_col2:
+ sort_option = st.selectbox("Sort by:", ["Relevance", "Name", "Size (Largest)", "Size (Smallest)"])
+ with filter_col3:
+ show_only_pdfs = st.checkbox("PDFs Only", value=False)
+
+ # Sort files based on selection
+ sorted_files = list(st.session_state.files)
+ if sort_option == "Name":
+ sorted_files.sort(key=lambda x: x['filename'])
+ elif sort_option == "Size (Largest)":
+ # Convert size strings to comparable values
+ def parse_size(size_str):
+ if 'Unknown' in size_str:
+ return 0
+ try:
+ value = float(size_str.split(' ')[0])
+ unit = size_str.split(' ')[1]
+ multipliers = {'bytes': 1, 'KB': 1024, 'MB': 1024**2, 'GB': 1024**3, 'TB': 1024**4}
+ return value * multipliers.get(unit, 0)
+ except:
+ return 0
+
+ sorted_files.sort(key=lambda x: parse_size(x['size']), reverse=True)
+ elif sort_option == "Size (Smallest)":
+ def parse_size(size_str):
+ if 'Unknown' in size_str:
+ return float('inf')
+ try:
+ value = float(size_str.split(' ')[0])
+ unit = size_str.split(' ')[1]
+ multipliers = {'bytes': 1, 'KB': 1024, 'MB': 1024**2, 'GB': 1024**3, 'TB': 1024**4}
+ return value * multipliers.get(unit, 0)
+ except:
+ return float('inf')
+
+ sorted_files.sort(key=lambda x: parse_size(x['size']))
+
+ # File list with selection
+ file_container = st.container()
+ with file_container:
+ selected_files = []
+ displayed_files = []
+
+ for i, file in enumerate(sorted_files):
+ # Apply filters
+ if file_filter and file_filter.lower() not in file['filename'].lower():
+ continue
+ if show_only_pdfs and not file['filename'].lower().endswith('.pdf'):
+ continue
+
+ displayed_files.append(i)
+ with st.container():
+ col1, col2, col3, col4 = st.columns([0.5, 3, 1, 1])
+ with col1:
+ selected = st.checkbox("", key=f"select_{i}", value=True)
+ if selected:
+ selected_files.append(i)
+ with col2:
+ file_icon = "๐"
+ if file['filename'].lower().endswith('.pdf'):
+ file_icon = "๐"
+ elif file['filename'].lower().endswith(('.doc', '.docx')):
+ file_icon = "๐"
+ elif file['filename'].lower().endswith(('.xls', '.xlsx')):
+ file_icon = "๐"
+ elif file['filename'].lower().endswith(('.ppt', '.pptx')):
+ file_icon = "๐ผ๏ธ"
+ elif file['filename'].lower().endswith(('.jpg', '.png', '.gif')):
+ file_icon = "๐ผ๏ธ"
+ elif file['filename'].lower().endswith(('.mp3', '.wav')):
+ file_icon = "๐"
+ elif file['filename'].lower().endswith(('.mp4', '.avi', '.mov')):
+ file_icon = "๐ฌ"
+
+ st.markdown(f"**{file_icon} {file['filename']}**")
+ st.markdown(f"{file['url'][:60]}...", unsafe_allow_html=True)
+ with col3:
+ st.markdown(f"**Size:** {file['size']}")
+ with col4:
+ st.button("Preview", key=f"preview_{i}")
+
+ st.divider()
+
+ if not displayed_files:
+ st.info("No files match your current filters. Try adjusting your search criteria.")
+
+ # Download options
+ if selected_files:
+ col1, col2 = st.columns(2)
+ with col1:
+ download_dir = st.text_input("Download Directory", value="downloads")
+ with col2:
+ download_option = st.radio("Download as", ["Individual Files", "ZIP Archive"], horizontal=True)
+
+ download_col1, download_col2, download_col3 = st.columns([3, 1, 1])
+ with download_col1:
+ download_button = st.button("โฌ๏ธ Download Selected Files", use_container_width=True)
+ with download_col2:
+ google_drive_button = st.button("๐ค Upload to Drive",
+ use_container_width=True,
+ disabled=not st.session_state.google_credentials)
+ with download_col3:
+ select_all = st.button("Select All Files", use_container_width=True)
+
+ # Handle select all button
+ if select_all:
+ for i in displayed_files:
+ st.session_state[f"select_{i}"] = True
+ st.rerun()
+
+ # Download progress/results
+ if st.session_state.download_complete:
+ st.success(f"โ
Downloaded {len(st.session_state.downloaded_paths)} files successfully!")
+ download_links = []
+ for path in st.session_state.downloaded_paths:
+ with open(path, "rb") as f:
+ file_content = f.read()
+ file_name = os.path.basename(path)
+ download_links.append((file_name, file_content))
+
+ if len(download_links) > 0:
+ if download_option == "ZIP Archive":
+ # Create ZIP archive for download
+ zip_path = create_zip_file(st.session_state.downloaded_paths, download_dir)
+ with open(zip_path, "rb") as f:
+ zip_content = f.read()
+ st.download_button("๐ฆ Download ZIP Archive",
+ zip_content,
+ file_name=os.path.basename(zip_path),
+ mime="application/zip")
+ else:
+ # Show individual file download links
+ st.markdown("Download Files
", unsafe_allow_html=True)
+
+ # Create a grid of download buttons
+ cols = st.columns(3)
+ for idx, (name, content) in enumerate(download_links):
+ mime_type = mimetypes.guess_type(name)[0] or 'application/octet-stream'
+ with cols[idx % 3]:
+ st.download_button(
+ f"๐ {name}",
+ content,
+ file_name=name,
+ mime=mime_type,
+ key=f"dl_{name}",
+ use_container_width=True
+ )
+
+ # Tab 2: Local File Search
+ with tabs[1]:
+ st.markdown("", unsafe_allow_html=True)
+ st.write("Upload files to search through their content with AI-powered semantic search.")
+
+ # File upload
+ uploaded_files = st.file_uploader("Upload documents for search",
+ accept_multiple_files=True,
+ type=['pdf', 'docx', 'txt', 'csv', 'json'])
+
+ if uploaded_files:
+ # Build search index on upload
+ col1, col2 = st.columns([4, 1])
+ with col1:
+ use_transformer = st.checkbox("Use AI Transformer Model", value=HAVE_TRANSFORMERS,
+ help="Uses advanced AI for more accurate semantic search (if available)")
+ with col2:
+ if st.button("Build Search Index", use_container_width=True):
+ with st.spinner("Processing files and building search index..."):
+ files_added = 0
+ for uploaded_file in uploaded_files:
+ file_info = {
+ 'filename': uploaded_file.name,
+ 'url': f'local://{uploaded_file.name}',
+ 'size': humanize_file_size(uploaded_file.size)
+ }
+ success = st.session_state.rag_search.add_file(uploaded_file.getvalue(), file_info)
+ if success:
+ files_added += 1
+
+ if files_added > 0:
+ index_built = st.session_state.rag_search.build_index()
+ if index_built:
+ st.success(f"โ
Successfully indexed {files_added} files!")
+ else:
+ st.error("Failed to build search index.")
+ else:
+ st.warning("No valid text could be extracted from the files.")
+
+ # Search interface
+ st.markdown("", unsafe_allow_html=True)
+
+ col1, col2 = st.columns([4, 1])
+ with col1:
+ query = st.text_input("Enter search query:", placeholder="e.g., neural networks, climate change")
+ with col2:
+ expand_query = st.checkbox("Auto-expand query", value=True,
+ help="Automatically add related terms to your search")
+
+ col1, col2 = st.columns([4, 1])
+ with col1:
+ if st.button("๐ Search Documents", use_container_width=True):
+ if not query:
+ st.warning("Please enter a search query")
+ else:
+ with st.spinner("Searching..."):
+ results = st.session_state.rag_search.search(query, top_k=5, search_chunks=True)
+
+ if results:
+ st.markdown(f"**Found {len(results)} relevant documents:**")
+ for i, result in enumerate(results):
+ with st.container():
+ st.markdown(f"", unsafe_allow_html=True)
+ st.markdown(f"**{i+1}. {result['file_info']['filename']}** (Score: {result['score']:.2f})")
+
+ if result.get('chunk_preview'):
+ st.markdown("**Matching content:**")
+ st.text(result['chunk_preview'])
+
+ st.markdown("
", unsafe_allow_html=True)
+ else:
+ st.info("No matching documents found. Try a different query.")
+ with col2:
+ num_results = st.number_input("Max results", min_value=1, max_value=20, value=5)
+
+ # Quick search tips
+ with st.expander("Search Tips", expanded=False):
+ st.markdown("""
+ ### Effective Search Tips
+
+ - **Be specific** with your queries for more accurate results
+ - **Try different phrasings** if you don't get the results you expect
+ - Use **quotation marks** for exact phrase matching
+ - For **complex topics**, break down your search into multiple queries
+ - **Combine related terms** to improve recall
+
+ The search engine uses advanced algorithms to understand the semantic meaning of your query,
+ not just keyword matching.
+ """)
+
+ # Tab 3: Advanced Configuration
+ with tabs[2]:
+ st.markdown("", unsafe_allow_html=True)
+
+ config_tabs = st.tabs(["Browser Settings", "Proxy Configuration", "Download Options", "System"])
+
+ # Browser Settings tab
+ with config_tabs[0]:
+ col1, col2 = st.columns(2)
+ with col1:
+ use_stealth = st.checkbox("Use Stealth Mode", value=st.session_state.stealth_mode,
+ help="Makes browser harder to detect as automated, but may be slower")
+
+ handle_captchas = st.checkbox("Handle Captchas Automatically", value=False,
+ help="Attempt to solve simple captchas automatically")
+
+ download_timeout = st.slider("Download Timeout (seconds)",
+ min_value=30, max_value=600, value=300,
+ help="Maximum time to wait for downloads to complete")
+ with col2:
+ user_agent = st.selectbox("User Agent", USER_AGENTS, index=0,
+ help="Browser identity to use when accessing websites")
+
+ save_screenshots = st.checkbox("Save Browser Screenshots", value=False,
+ help="Save screenshots when errors occur for debugging")
+
+ browser_lang = st.selectbox("Browser Language",
+ ["English (US)", "English (UK)", "Spanish", "French", "German", "Chinese"],
+ index=0)
+
+ if st.button("Update Browser Settings"):
+ st.session_state.stealth_mode = use_stealth
+ st.success("Browser settings updated!")
+
+ # Dependency installation section
+ st.markdown("", unsafe_allow_html=True)
+ if st.button("Install Playwright Dependencies"):
+ with st.spinner("Installing dependencies..."):
+ install_playwright_dependencies()
+
+ # Proxy Configuration tab
+ with config_tabs[1]:
+ proxy_enabled = st.checkbox("Enable Proxy", value=st.session_state.use_proxy,
+ help="Route requests through a proxy server for anonymity or bypassing restrictions")
+
+ if proxy_enabled:
+ proxy_col1, proxy_col2 = st.columns(2)
+ with proxy_col1:
+ proxy_type = st.selectbox("Proxy Type", ["HTTP", "SOCKS5", "HTTPS"])
+ proxy_host = st.text_input("Proxy Host", placeholder="e.g., 127.0.0.1")
+ with proxy_col2:
+ proxy_port = st.text_input("Proxy Port", placeholder="e.g., 8080")
+ proxy_auth = st.text_input("Proxy Authentication (optional)",
+ placeholder="username:password", type="password")
+
+ st.markdown("", unsafe_allow_html=True)
+ use_proxy_rotation = st.checkbox("Enable Proxy Rotation", value=False,
+ help="Automatically rotate between multiple proxies for better anonymity")
+
+ if use_proxy_rotation:
+ proxy_list = st.text_area("Proxy List (one per line)",
+ placeholder="http://proxy1.example.com:8080\nhttp://proxy2.example.com:8080")
+ rotation_interval = st.slider("Rotation Interval (requests)",
+ min_value=1, max_value=50, value=10,
+ help="How often to switch proxies")
+
+ if st.button("Save Proxy Configuration"):
+ # Construct the proxy string
+ proxy_string = None
+ if proxy_enabled and proxy_host and proxy_port:
+ proxy_prefix = f"{proxy_type.lower()}://"
+ proxy_auth_str = f"{proxy_auth}@" if proxy_auth else ""
+ proxy_string = f"{proxy_prefix}{proxy_auth_str}{proxy_host}:{proxy_port}"
+
+ # Update session state
+ st.session_state.use_proxy = proxy_enabled
+ st.session_state.proxy_string = proxy_string
+
+ # Configure proxy rotation if enabled
+ if use_proxy_rotation and proxy_list:
+ PROXY_ROTATION_CONFIG["enabled"] = True
+ PROXY_ROTATION_CONFIG["rotation_interval"] = rotation_interval
+ PROXY_ROTATION_CONFIG["proxies"] = [p.strip() for p in proxy_list.splitlines() if p.strip()]
+
+ st.success("Proxy configuration updated!")
+
+ # Download Options tab
+ with config_tabs[2]:
+ col1, col2 = st.columns(2)
+ with col1:
+ st.markdown("", unsafe_allow_html=True)
+
+ skip_existing = st.checkbox("Skip Existing Files", value=True,
+ help="Don't download files that already exist locally")
+
+ auto_rename = st.checkbox("Auto-Rename Duplicates", value=True,
+ help="Automatically rename files instead of overwriting")
+
+ verify_downloads = st.checkbox("Verify Downloads", value=True,
+ help="Check file integrity after download")
+
+ max_retries = st.slider("Max Retries", min_value=0, max_value=10, value=3,
+ help="Number of times to retry failed downloads")
+
+ with col2:
+ st.markdown("", unsafe_allow_html=True)
+
+ auto_organize = st.checkbox("Auto-Organize Files", value=True,
+ help="Automatically organize files by type")
+
+ default_dir = st.text_input("Default Download Directory", value="downloads",
+ help="Default location to save downloaded files")
+
+ org_by_domain = st.checkbox("Organize by Domain", value=False,
+ help="Create subdirectories based on source domains")
+
+ org_by_type = st.checkbox("Organize by File Type", value=False,
+ help="Create subdirectories based on file types")
+
+ if st.button("Save Download Settings"):
+ st.session_state.download_settings = {
+ "skip_existing": skip_existing,
+ "auto_rename": auto_rename,
+ "verify_downloads": verify_downloads,
+ "max_retries": max_retries,
+ "auto_organize": auto_organize,
+ "default_dir": default_dir,
+ "org_by_domain": org_by_domain,
+ "org_by_type": org_by_type
+ }
+ st.success("Download settings saved!")
+
+ # System tab
+ with config_tabs[3]:
+ col1, col2 = st.columns(2)
+ with col1:
+ st.markdown("", unsafe_allow_html=True)
+
+ max_concurrent = st.slider("Max Concurrent Downloads", min_value=1, max_value=10, value=3,
+ help="Maximum number of simultaneous downloads")
+
+ memory_limit = st.slider("Memory Limit (MB)", min_value=256, max_value=4096, value=1024,
+ help="Maximum memory to use for file processing")
+
+ processing_threads = st.slider("Processing Threads", min_value=1, max_value=8, value=2,
+ help="Number of threads to use for file processing")
+
+ with col2:
+ st.markdown("", unsafe_allow_html=True)
+
+ log_level = st.selectbox("Log Level", ["DEBUG", "INFO", "WARNING", "ERROR"], index=1,
+ help="Detail level for application logs")
+
+ save_debug_info = st.checkbox("Save Debug Information", value=False,
+ help="Save detailed information about program execution")
+
+ log_dir = st.text_input("Log Directory", value="logs",
+ help="Directory to save log files")
+
+ if st.button("Apply System Settings"):
+ st.session_state.system_settings = {
+ "max_concurrent": max_concurrent,
+ "memory_limit": memory_limit,
+ "processing_threads": processing_threads,
+ "log_level": log_level,
+ "save_debug_info": save_debug_info,
+ "log_dir": log_dir
+ }
+ # Update logging configuration
+ log_level_num = getattr(logging, log_level)
+ logging.getLogger().setLevel(log_level_num)
+ st.success("System settings applied!")
+
+ # Reset application button
+ st.markdown("", unsafe_allow_html=True)
+ reset_col1, reset_col2 = st.columns([1, 3])
+ with reset_col1:
+ if st.button("Reset Application", use_container_width=True):
+ for key in list(st.session_state.keys()):
+ if key != 'google_credentials': # Preserve Google auth
+ del st.session_state[key]
+ st.success("Application has been reset!")
+ st.rerun()
+ with reset_col2:
+ st.info("This will clear all search results, downloaded files, and reset settings to defaults.")
+
+ # Advanced cleanup options
+ st.markdown("", unsafe_allow_html=True)
+
+ adv_col1, adv_col2 = st.columns(2)
+ with adv_col1:
+ clear_cache = st.button("Clear Cache", use_container_width=True)
+ if clear_cache:
+ # Clear cached files and temporary data
+ temp_dir = tempfile.gettempdir()
+ try:
+ for f in os.listdir(temp_dir):
+ if f.startswith("playwright") or f.startswith("download"):
+ try:
+ os.remove(os.path.join(temp_dir, f))
+ except:
+ pass
+ st.success("Cache cleared successfully!")
+ except Exception as e:
+ st.error(f"Error clearing cache: {e}")
+
+ with adv_col2:
+ export_settings = st.button("Export Settings", use_container_width=True)
+ if export_settings:
+ # Export current settings to JSON
+ settings = {
+ "mode": st.session_state.mode,
+ "stealth_mode": st.session_state.stealth_mode,
+ "use_proxy": st.session_state.use_proxy,
+ "proxy_string": st.session_state.proxy_string,
+ "custom_extensions": st.session_state.get("custom_extensions", ""),
+ "prioritize_pdfs": st.session_state.get("prioritize_pdfs", True),
+ "system_settings": st.session_state.get("system_settings", {}),
+ "download_settings": st.session_state.get("download_settings", {})
+ }
+
+ settings_json = json.dumps(settings, indent=2)
+ b64 = base64.b64encode(settings_json.encode()).decode()
+ href = f'data:application/json;base64,{b64}'
+ st.markdown(f'