utils.py

import os
import random
import re
import zipfile
import datetime
import logging
import mimetypes
from urllib.parse import urlparse, parse_qs, quote, unquote
import requests

# Setup logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler('app.log'),
        logging.StreamHandler()
    ]
)
logger = logging.getLogger(__name__)

# User agent list
USER_AGENTS = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.54',
    'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1',
    'Mozilla/5.0 (iPad; CPU OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0'
]

# Stealth browser settings
STEALTH_SETTINGS = {
    "hardware_concurrency": 4,
    "device_memory": 8,
    "webgl_vendor": "Google Inc. (Intel)",
    "webgl_renderer": "Intel Iris OpenGL Engine",
    "languages": ["en-US", "en"],
    "disable_webrtc": True,
    "navigator_platform": "Win32",
    "touch_support": False
}

# Proxy rotation configuration
PROXY_ROTATION_CONFIG = {
    "enabled": False,
    "rotation_interval": 10,
    "proxies": []
}

def get_random_user_agent():
    """Return a random user agent from the list"""
    return random.choice(USER_AGENTS)

def sizeof_fmt(num, suffix='B'):
    """Format file size in human-readable format"""
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if abs(num) < 1024.0:
            return f"{num:3.1f}{unit}{suffix}"
        num /= 1024.0
    return f"{num:.1f}Y{suffix}"

def create_zip_file(file_paths, output_dir):
    """Create a ZIP file containing the given files"""
    timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
    zip_path = os.path.join(output_dir, f"downloads_{timestamp}.zip")
    with zipfile.ZipFile(zip_path, 'w') as zipf:
        for file_path in file_paths:
            zipf.write(file_path, os.path.basename(file_path))
    return zip_path

def get_file_extension(url, default='.pdf'):
    """Extract file extension from URL or filename"""
    path = urlparse(url).path
    ext = os.path.splitext(path)[1].lower()
    if not ext:
        return default
    return ext

def humanize_file_size(size_bytes):
    """Format file size in human-readable format"""
    if size_bytes < 1024:
        return f"{size_bytes} bytes"
    for unit in ['KB', 'MB', 'GB', 'TB']:
        size_bytes /= 1024.0
        if size_bytes < 1024.0:
            return f"{size_bytes:.1f} {unit}"
    return f"{size_bytes:.1f} PB"

def get_domain(url):
    """Extract domain from URL"""
    parsed = urlparse(url)
    return parsed.netloc

def is_valid_file_url(url, extensions):
    """Check if URL is a valid file URL based on extension"""
    return any(url.lower().endswith(ext) for ext in extensions)

def detect_captcha(html_content):
    """Detect common captcha patterns in HTML content"""
    captcha_patterns = [
        'captcha', 'recaptcha', 'g-recaptcha', 'hcaptcha', 'cf-turnstile',
        'challenge', 'solve the following', 'verify you are human'
    ]
    html_lower = html_content.lower()
    return any(pattern in html_lower for pattern in captcha_patterns)

def is_download_link(url):
    """Enhanced function to detect if a URL is likely a download link"""
    url_lower = url.lower()
    
    # Check for common download-related terms
    download_terms = [
        'download', 'dl', 'get', 'file', 'attachment', 'export', 'view',
        'retrieve', 'fetch', 'load', 'open', 'access', 'doc', 'document'
    ]
    if any(term in url_lower for term in download_terms):
        return True
    
    # Check for common download script patterns
    script_patterns = [
        'download.php', 'getfile.php', 'fetch.php', 'view.php', 'dl.php',
        'download.aspx', 'getfile.aspx', 'file.aspx',
        'downloadhandler', 'filehandler', 'filedownload',
        'download.jsp', 'download.cgi', 'download.do',
        'download-file', 'get-file',
        'downloadfile', 'getfile', 'viewfile',
        'Action=downloadfile', 'action=download', 'action=view',
        'download?', 'file?', 'get?', 'view?'
    ]
    if any(pattern in url_lower for pattern in script_patterns):
        return True
    
    # Check for common file extensions
    path = urlparse(url).path
    common_extensions = ['.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', 
                         '.zip', '.rar', '.txt', '.csv', '.json', '.xml', '.jpg', 
                         '.png', '.gif', '.mp3', '.mp4', '.avi', '.mov']
    
    if any(ext in path.lower() for ext in common_extensions):
        return True
    
    # Check for file parameters in URL
    params = parse_qs(urlparse(url).query)
    param_keys = params.keys()
    file_param_indicators = ['file', 'id', 'key', 'filename', 'name', 'fileid', 'attachment', 'attid']
    if any(key.lower() in file_param_indicators for key in param_keys):
        return True
    
    # Check for complex encoding patterns
    if 'Action=downloadfile' in url or 'fname=' in url:
        return True
        
    return False

def normalize_download_url(url):
    """Normalize download URLs to handle various formats and encodings"""
    try:
        parsed = urlparse(url)
        
        # Handle phpMyAdmin-style encoded URLs
        if 'Action=downloadfile' in url and 'file=' in url:
            # Keep the URL as is for now
            return url
        
        # Handle URLs with fname parameter
        if 'fname=' in url:
            return url
            
        # Quote the path portion if needed
        path = parsed.path
        if '%' not in path and ' ' in path:
            path = quote(path)
            
        # Reconstruct the URL
        normalized = parsed._replace(path=path).geturl()
        return normalized
    except Exception as e:
        logger.error(f"Error normalizing URL {url}: {e}")
        return url

def show_user_friendly_error(error_type, details, suggestion=None):
    """Display a user-friendly error message with suggestions"""
    import streamlit as st
    with st.error(f"**{error_type}**"):
        st.write(details)
        if suggestion:
            st.info(f"**Suggestion**: {suggestion}")