import os import random import re import zipfile import datetime import logging import mimetypes from urllib.parse import urlparse, parse_qs, quote, unquote import requests # Setup logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler('app.log'), logging.StreamHandler() ] ) logger = logging.getLogger(__name__) # User agent list USER_AGENTS = [ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.54', 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1', 'Mozilla/5.0 (iPad; CPU OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0' ] # Stealth browser settings STEALTH_SETTINGS = { "hardware_concurrency": 4, "device_memory": 8, "webgl_vendor": "Google Inc. (Intel)", "webgl_renderer": "Intel Iris OpenGL Engine", "languages": ["en-US", "en"], "disable_webrtc": True, "navigator_platform": "Win32", "touch_support": False } # Proxy rotation configuration PROXY_ROTATION_CONFIG = { "enabled": False, "rotation_interval": 10, "proxies": [] } def get_random_user_agent(): """Return a random user agent from the list""" return random.choice(USER_AGENTS) def sizeof_fmt(num, suffix='B'): """Format file size in human-readable format""" for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: if abs(num) < 1024.0: return f"{num:3.1f}{unit}{suffix}" num /= 1024.0 return f"{num:.1f}Y{suffix}" def create_zip_file(file_paths, output_dir): """Create a ZIP file containing the given files""" timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") zip_path = os.path.join(output_dir, f"downloads_{timestamp}.zip") with zipfile.ZipFile(zip_path, 'w') as zipf: for file_path in file_paths: zipf.write(file_path, os.path.basename(file_path)) return zip_path def get_file_extension(url, default='.pdf'): """Extract file extension from URL or filename""" path = urlparse(url).path ext = os.path.splitext(path)[1].lower() if not ext: return default return ext def humanize_file_size(size_bytes): """Format file size in human-readable format""" if size_bytes < 1024: return f"{size_bytes} bytes" for unit in ['KB', 'MB', 'GB', 'TB']: size_bytes /= 1024.0 if size_bytes < 1024.0: return f"{size_bytes:.1f} {unit}" return f"{size_bytes:.1f} PB" def get_domain(url): """Extract domain from URL""" parsed = urlparse(url) return parsed.netloc def is_valid_file_url(url, extensions): """Check if URL is a valid file URL based on extension""" return any(url.lower().endswith(ext) for ext in extensions) def detect_captcha(html_content): """Detect common captcha patterns in HTML content""" captcha_patterns = [ 'captcha', 'recaptcha', 'g-recaptcha', 'hcaptcha', 'cf-turnstile', 'challenge', 'solve the following', 'verify you are human' ] html_lower = html_content.lower() return any(pattern in html_lower for pattern in captcha_patterns) def is_download_link(url): """Enhanced function to detect if a URL is likely a download link""" url_lower = url.lower() # Check for common download-related terms download_terms = [ 'download', 'dl', 'get', 'file', 'attachment', 'export', 'view', 'retrieve', 'fetch', 'load', 'open', 'access', 'doc', 'document' ] if any(term in url_lower for term in download_terms): return True # Check for common download script patterns script_patterns = [ 'download.php', 'getfile.php', 'fetch.php', 'view.php', 'dl.php', 'download.aspx', 'getfile.aspx', 'file.aspx', 'downloadhandler', 'filehandler', 'filedownload', 'download.jsp', 'download.cgi', 'download.do', 'download-file', 'get-file', 'downloadfile', 'getfile', 'viewfile', 'Action=downloadfile', 'action=download', 'action=view', 'download?', 'file?', 'get?', 'view?' ] if any(pattern in url_lower for pattern in script_patterns): return True # Check for common file extensions path = urlparse(url).path common_extensions = ['.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.zip', '.rar', '.txt', '.csv', '.json', '.xml', '.jpg', '.png', '.gif', '.mp3', '.mp4', '.avi', '.mov'] if any(ext in path.lower() for ext in common_extensions): return True # Check for file parameters in URL params = parse_qs(urlparse(url).query) param_keys = params.keys() file_param_indicators = ['file', 'id', 'key', 'filename', 'name', 'fileid', 'attachment', 'attid'] if any(key.lower() in file_param_indicators for key in param_keys): return True # Check for complex encoding patterns if 'Action=downloadfile' in url or 'fname=' in url: return True return False def normalize_download_url(url): """Normalize download URLs to handle various formats and encodings""" try: parsed = urlparse(url) # Handle phpMyAdmin-style encoded URLs if 'Action=downloadfile' in url and 'file=' in url: # Keep the URL as is for now return url # Handle URLs with fname parameter if 'fname=' in url: return url # Quote the path portion if needed path = parsed.path if '%' not in path and ' ' in path: path = quote(path) # Reconstruct the URL normalized = parsed._replace(path=path).geturl() return normalized except Exception as e: logger.error(f"Error normalizing URL {url}: {e}") return url def show_user_friendly_error(error_type, details, suggestion=None): """Display a user-friendly error message with suggestions""" import streamlit as st with st.error(f"**{error_type}**"): st.write(details) if suggestion: st.info(f"**Suggestion**: {suggestion}")