import streamlit as st # Must be the first Streamlit command at the global level st.set_page_config(page_title="Advanced File Downloader", layout="wide") # Now all other imports import os import subprocess from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError import asyncio import logging from urllib.parse import urlparse import re from pathlib import Path from io import BytesIO import random from bs4 import BeautifulSoup from PyPDF2 import PdfReader import zipfile import tempfile import mimetypes import requests import spacy import spacy.cli from spacy.language import Language from sentence_transformers import SentenceTransformer, util from transformers import pipeline # Initialize logging logging.basicConfig( filename='advanced_download_log.txt', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s' ) logger = logging.getLogger() # Model initialization with caching @st.cache_resource def initialize_models(): # spaCy try: nlp = spacy.load("en_core_web_sm") except OSError: spacy.cli.download("en_core_web_sm") nlp = spacy.load("en_core_web_sm") # SentenceTransformer semantic_model = SentenceTransformer('all-MiniLM-L6-v2') # Transformers summarizer = pipeline("summarization") return nlp, semantic_model, summarizer # Initialize models nlp_model, semantic_model, summarizer = initialize_models() # Rest of your code... # Also load SentenceTransformer for semantic re-ranking. from sentence_transformers import SentenceTransformer, util @st.cache_resource def load_semantic_model(): return SentenceTransformer('all-MiniLM-L6-v2') semantic_model = load_semantic_model() # -------------------- Transformers Summarization Setup -------------------- from transformers import pipeline @st.cache_resource def load_summarizer(): return pipeline("summarization") summarizer = load_summarizer() def summarize_pdf_url(pdf_url): """ Downloads a PDF from the given URL, extracts text using PyPDF2, and returns a summary of (up to) the first 3000 characters. """ try: with st.spinner("Downloading and processing PDF..."): response = requests.get(pdf_url, stream=True) temp_pdf = tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") with open(temp_pdf.name, "wb") as f: f.write(response.content) reader = PdfReader(temp_pdf.name) text = " ".join([page.extract_text() or "" for page in reader.pages]) os.remove(temp_pdf.name) limited_text = text[:3000] # Limit text for summarization summary = summarizer(limited_text, max_length=200, min_length=50, do_sample=False) return summary[0]["summary_text"] except Exception as e: return f"Error summarizing PDF: {e}" # -------------------- Google API Setup -------------------- GOOGLE_OAUTH_CONFIG = { "web": { "client_id": "90798824947-u25obg1q844qeikjoh4jdmi579kn9p1c.apps.googleusercontent.com", "project_id": "huggingface-449214", "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "client_secret": "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", "redirect_uris": ["https://euler314-craw-web.hf.space/"] } } import google_auth_oauthlib.flow import googleapiclient.discovery import google.auth.transport.requests def get_google_auth_url(): client_config = GOOGLE_OAUTH_CONFIG["web"] flow = google_auth_oauthlib.flow.Flow.from_client_config( {"web": client_config}, scopes=["https://www.googleapis.com/auth/drive.file"] ) flow.redirect_uri = client_config["redirect_uris"][0] authorization_url, _ = flow.authorization_url( access_type="offline", include_granted_scopes="true", prompt="consent" ) return authorization_url def exchange_code_for_credentials(auth_code): if not auth_code.strip(): return None, "No code provided." try: client_config = GOOGLE_OAUTH_CONFIG["web"] flow = google_auth_oauthlib.flow.Flow.from_client_config( {"web": client_config}, scopes=["https://www.googleapis.com/auth/drive.file"] ) flow.redirect_uri = client_config["redirect_uris"][0] flow.fetch_token(code=auth_code.strip()) creds = flow.credentials if not creds or not creds.valid: return None, "Could not validate credentials. Check code and try again." return creds, "Google Sign-In successful!" except Exception as e: return None, f"Error during token exchange: {e}" # -------------------- Playwright Setup -------------------- def install_playwright_dependencies(): os.environ['PLAYWRIGHT_BROWSERS_PATH'] = os.path.expanduser("~/.cache/ms-playwright") os.environ['LD_LIBRARY_PATH'] = '/usr/lib/playwright:/usr/lib/x86_64-linux-gnu' try: subprocess.run(['apt-get', 'update', '-y'], check=True) packages = [ 'libnss3', 'libnss3-tools', 'libnspr4', 'libatk1.0-0', 'libatk-bridge2.0-0', 'libatspi2.0-0', 'libcups2', 'libxcomposite1', 'libxdamage1', 'libdrm2', 'libgbm1', 'libpango-1.0-0' ] subprocess.run(['apt-get', 'install', '-y', '--no-install-recommends'] + packages, check=True) os.makedirs('/usr/lib/playwright', exist_ok=True) symlinks = { 'libnss3.so': '/usr/lib/x86_64-linux-gnu/libnss3.so', 'libnssutil3.so': '/usr/lib/x86_64-linux-gnu/libnssutil3.so', 'libsmime3.so': '/usr/lib/x86_64-linux-gnu/libsmime3.so', 'libnspr4.so': '/usr/lib/x86_64-linux-gnu/libnspr4.so', 'libatk-1.0.so.0': '/usr/lib/x86_64-linux-gnu/libatk-1.0.so.0', 'libatk-bridge-2.0.so.0': '/usr/lib/x86_64-linux-gnu/libatk-bridge-2.0.so.0', 'libcups.so.2': '/usr/lib/x86_64-linux-gnu/libcups.so.2', 'libatspi.so.0': '/usr/lib/x86_64-linux-gnu/libatspi.so.0', 'libXcomposite.so.1': '/usr/lib/x86_64-linux-gnu/libXcomposite.so.1', 'libXdamage.so.1': '/usr/lib/x86_64-linux-gnu/libXdamage.so.1' } for link_name, target in symlinks.items(): link_path = os.path.join('/usr/lib/playwright', link_name) if not os.path.exists(link_path): os.symlink(target, link_path) subprocess.run(['python3', '-m', 'playwright', 'install', 'chromium'], check=True) browser_path = os.path.expanduser("~/.cache/ms-playwright") os.makedirs(browser_path, exist_ok=True) subprocess.run(['chmod', '-R', '755', browser_path], check=True) except subprocess.CalledProcessError as e: st.error(f"Error installing dependencies: {e}") except Exception as e: st.error(f"Error: {e}") # Initialize Playwright dependencies install_playwright_dependencies() # -------------------- Logging Setup -------------------- logging.basicConfig( filename='advanced_download_log.txt', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s' ) logger = logging.getLogger() # -------------------- Shared Utils -------------------- USER_AGENTS = [ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0', ] def get_random_user_agent(): return random.choice(USER_AGENTS) def sizeof_fmt(num, suffix='B'): for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: if abs(num) < 1024.0: return f"{num:3.1f}{unit}{suffix}" num /= 1024.0 return f"{num:.1f}Y{suffix}" # ---------- Human-like Interactions ------------- async def human_like_scroll(page): scroll_height = await page.evaluate('document.body.scrollHeight') viewport_height = await page.evaluate('window.innerHeight') current_scroll = 0 while current_scroll < scroll_height: await page.evaluate(f'window.scrollTo(0, {current_scroll})') await asyncio.sleep(random.uniform(0.5, 1.5)) current_scroll += viewport_height * random.uniform(0.5, 1.5) scroll_height = await page.evaluate('document.body.scrollHeight') async def human_like_interactions(page): await page.mouse.move(random.randint(0, 1000), random.randint(0, 1000)) await asyncio.sleep(random.uniform(0.5, 1.5)) await page.mouse.click(random.randint(0, 1000), random.randint(0, 1000)) await asyncio.sleep(random.uniform(0.5, 1.5)) await page.evaluate("window.scrollBy(0, window.innerHeight / 2)") await asyncio.sleep(random.uniform(0.5, 1.5)) # ---------- NLP Helpers ------------- def nlp_preprocess(query: str) -> str: doc = nlp_model(query) tokens = [token.lemma_.lower() for token in doc if not token.is_stop and token.is_alpha] processed = " ".join(tokens) return processed if processed.strip() else query def nlp_extract_entities(text: str): doc = nlp_model(text) return [(ent.text, ent.label_) for ent in doc.ents] # ---------- AI-enhanced Query Preprocessing ------------- def ai_preprocess_query(query: str) -> str: return query # ---------- Download Manager ------------- class DownloadManager: def __init__(self, use_proxy=False, proxy=None, query=None, num_results=5): self.use_proxy = use_proxy self.proxy = proxy self.query = query self.num_results = num_results self.playwright = None self.browser = None self.context = None self.page = None async def __aenter__(self): self.playwright = await async_playwright().start() opts = {"headless": True} if self.use_proxy and self.proxy: opts["proxy"] = {"server": self.proxy} self.browser = await self.playwright.chromium.launch(**opts) self.context = await self.browser.new_context(user_agent=get_random_user_agent()) self.page = await self.context.new_page() await self.page.set_extra_http_headers({ 'Accept-Language': 'en-US,en;q=0.9', 'Accept-Encoding': 'gzip, deflate, br', 'Referer': 'https://www.bing.com/' }) return self async def __aexit__(self, exc_type, exc_val, exc_tb): if self.browser: await self.browser.close() if self.playwright: await self.playwright.stop() async def get_file_size(self, url): try: response = await self.page.request.head(url) length = response.headers.get('Content-Length', None) if length: return sizeof_fmt(int(length)) else: return "Unknown Size" except Exception: return "Unknown Size" async def get_pdf_metadata(self, url): try: resp = await self.page.request.get(url, timeout=15000) if resp.ok: content = await resp.body() pdf = BytesIO(content) reader = PdfReader(pdf) return { 'Title': reader.metadata.title if reader.metadata.title else 'N/A', 'Author': reader.metadata.author if reader.metadata.author else 'N/A', 'Pages': len(reader.pages), } else: return {} except Exception: return {} async def search_bing(self): if not self.query: return [], [] query = self.query if "filetype:pdf" not in query.lower(): query += " filetype:pdf" if "site:" not in query.lower(): query += " site:edu OR site:arxiv.org OR site:openstax.org" query = ai_preprocess_query(query) query_processed = nlp_preprocess(query) logger.info(f"BING SEARCH NLP: Original='{query}' -> Processed='{query_processed}'") bing_url = f"https://www.bing.com/search?q={query_processed.replace(' ', '+')}&count={self.num_results}" try: await self.page.goto(bing_url, timeout=30000) await self.page.wait_for_selector('li.b_algo', timeout=30000) await human_like_scroll(self.page) html = await self.page.content() soup = BeautifulSoup(html, 'html.parser') raw_results = soup.find_all('li', class_='b_algo') url_list = [] info_list = [] snippets = [] for r in raw_results: link_tag = r.find('a') snippet_tag = r.find('p') snippet_text = snippet_tag.get_text(strip=True) if snippet_tag else "" snippets.append(snippet_text) entities = nlp_extract_entities(snippet_text) if link_tag and 'href' in link_tag.attrs: link_url = link_tag['href'] url_list.append(link_url) info_list.append({ 'url': link_url, 'snippet': snippet_text, 'entities': entities }) if len(url_list) >= self.num_results: break query_emb = semantic_model.encode(query, convert_to_tensor=True) snippet_embs = semantic_model.encode(snippets, convert_to_tensor=True) scores = util.cos_sim(query_emb, snippet_embs)[0] sorted_indices = scores.argsort(descending=True).cpu().numpy().tolist() sorted_url_list = [url_list[i] for i in sorted_indices] sorted_info_list = [info_list[i] for i in sorted_indices] return sorted_url_list, sorted_info_list except PlaywrightTimeoutError: logger.error("Bing search timed out.") return [], [] except Exception as e: logger.error(f"Bing search error: {e}") return [], [] async def extract_downloadable_files(self, url, custom_ext_list): found_files = [] try: await self.page.goto(url, timeout=30000) await self.page.wait_for_load_state('networkidle', timeout=30000) await human_like_interactions(self.page) content = await self.page.content() soup = BeautifulSoup(content, 'html.parser') default_exts = [ '.pdf', '.docx', '.zip', '.rar', '.exe', '.mp3', '.mp4', '.avi', '.mkv', '.png', '.jpg', '.jpeg', '.gif' ] all_exts = set(default_exts + [ext.strip().lower() for ext in custom_ext_list if ext.strip()]) anchors = soup.find_all('a', href=True) for a in anchors: href = a['href'].strip() if any(href.lower().endswith(ext) for ext in all_exts): if href.startswith('http'): file_url = href elif href.startswith('/'): parsed = urlparse(url) file_url = f"{parsed.scheme}://{parsed.netloc}{href}" else: continue size_str = await self.get_file_size(file_url) meta = {} if file_url.lower().endswith('.pdf'): meta = await self.get_pdf_metadata(file_url) found_files.append({ 'url': file_url, 'filename': os.path.basename(file_url.split('?')[0]), 'size': size_str, 'metadata': meta }) elif ("drive.google.com" in href) or ("drive.com" in href): file_id = None for pattern in [ r'/file/d/([^/]+)/', r'open\?id=([^&]+)', r'id=([^&]+)' ]: match = re.search(pattern, href) if match: file_id = match.group(1) break if file_id: direct = f"https://drive.google.com/uc?export=download&id={file_id}" filename = f"drive_file_{file_id}" try: resp = await self.page.request.head(direct, timeout=15000) cd = resp.headers.get("Content-Disposition", "") if cd: mt = re.search(r'filename\*?="?([^";]+)', cd) if mt: filename = mt.group(1).strip('"').strip() else: ctype = resp.headers.get("Content-Type", "") ext_guess = mimetypes.guess_extension(ctype) or "" filename = f"drive_file_{file_id}{ext_guess}" except Exception: pass size_str = await self.get_file_size(direct) found_files.append({ 'url': direct, 'filename': filename, 'size': size_str, 'metadata': {} }) return found_files except PlaywrightTimeoutError: logger.error(f"Timeout extracting from {url}") return [] except Exception as e: logger.error(f"Error extracting from {url}: {e}") return [] async def download_file(self, file_info, save_dir, referer): file_url = file_info['url'] fname = file_info['filename'] path = os.path.join(save_dir, fname) base, ext = os.path.splitext(fname) i = 1 while os.path.exists(path): path = os.path.join(save_dir, f"{base}({i}){ext}") i += 1 os.makedirs(save_dir, exist_ok=True) try: if file_url.lower().endswith(".pdf") and "drive.google.com" not in file_url.lower(): response = requests.get(file_url, stream=True) with open(path, "wb") as f: f.write(response.content) logger.info(f"Directly downloaded PDF: {path}") return path if "drive.google.com" in file_url.lower(): import gdown try: result = gdown.download(file_url, output=path, quiet=False, fuzzy=True) if result is None: logger.error(f"gdown failed to download: {file_url}") return None current_ext = os.path.splitext(path)[1].lower() allowed_exts = {'.pdf', '.jpg', '.jpeg', '.png', '.docx', '.zip', '.rar', '.mp3', '.mp4', '.avi', '.mkv'} if current_ext not in allowed_exts: try: r = requests.head(file_url, allow_redirects=True, timeout=15) ctype = r.headers.get("Content-Type", "") guessed_ext = mimetypes.guess_extension(ctype) or ".pdf" except Exception as e: logger.error(f"Error in HEAD request for extension: {e}") guessed_ext = ".pdf" new_path = os.path.splitext(path)[0] + guessed_ext os.rename(path, new_path) path = new_path logger.info(f"Downloaded using gdown: {path}") return path except Exception as e: logger.error(f"Error downloading using gdown: {e}") return None headers = { 'Accept-Language': 'en-US,en;q=0.9', 'Accept-Encoding': 'gzip, deflate, br', 'Referer': referer } await human_like_interactions(self.page) resp = await self.page.request.get(file_url, headers=headers, timeout=30000) if resp.status == 403: logger.error(f"403 Forbidden: {file_url}") return None if not resp.ok: logger.error(f"Failed to download {file_url}: Status {resp.status}") return None data = await resp.body() with open(path, 'wb') as f: f.write(data) logger.info(f"Downloaded: {path}") return path except PlaywrightTimeoutError: logger.error(f"Timeout downloading {file_url}") return None except Exception as e: logger.error(f"Error downloading {file_url}: {e}") return None async def deep_search(self, url, custom_ext_list, sublink_limit=2000, max_concurrency=500): progress_text = st.empty() progress_bar = st.progress(0) progress_text.text("Analyzing main page...") all_files = [] main_files = await self.extract_downloadable_files(url, custom_ext_list) all_files.extend(main_files) progress_text.text("Getting sublinks...") sublinks = await self.get_sublinks(url, sublink_limit) total_links = len(sublinks) progress_text.text(f"Processing {total_links} sublinks...") sem = asyncio.Semaphore(max_concurrency) async def analyze_one_sublink(link, idx): async with sem: progress_text.text(f"Processing link {idx}/{total_links}: {link}") progress_bar.progress(idx/total_links) return await self.extract_downloadable_files(link, custom_ext_list) tasks = [analyze_one_sublink(link, i) for i, link in enumerate(sublinks, 1)] sub_results = await asyncio.gather(*tasks) for sr in sub_results: all_files.extend(sr) unique_map = {f['url']: f for f in all_files} combined = list(unique_map.values()) progress_text.text(f"Found {len(combined)} unique files.") progress_bar.progress(1.0) return combined async def get_sublinks(self, url, limit=20000): try: await self.page.goto(url, timeout=30000) content = await self.page.content() soup = BeautifulSoup(content, "html.parser") links = [] for a in soup.find_all('a', href=True): href = a['href'].strip() if href.startswith('http'): links.append(href) elif href.startswith('/'): parsed = urlparse(url) links.append(f"{parsed.scheme}://{parsed.netloc}{href}") return list(set(links))[:limit] except Exception as e: logger.error(f"Error getting sublinks: {e}") return [] def main(): st.set_page_config(page_title="Advanced File Downloader", layout="wide") if 'session_state' not in st.session_state: st.session_state.session_state = { 'discovered_files': [], 'current_url': None, 'download_manager': None, 'google_creds': None } st.title("Advanced File Downloader") mode = st.sidebar.radio("Select Mode", ["Manual URL", "Bing Search", "PDF Summarizer"]) with st.sidebar.expander("Advanced Options"): custom_extensions = st.text_input( "Custom File Extensions", placeholder=".csv, .txt, .epub" ) max_concurrency = st.slider( "Max Concurrency", min_value=1, max_value=1000, value=200 ) use_proxy = st.checkbox("Use Proxy") proxy = st.text_input("Proxy URL", placeholder="http://proxy:port") # Google OAuth Section with st.expander("Google Drive Integration"): if st.button("Start Google Sign-In"): auth_url = get_google_auth_url() st.markdown(f"[Click here to authorize]({auth_url})") auth_code = st.text_input("Enter authorization code") if st.button("Complete Sign-In") and auth_code: creds, msg = exchange_code_for_credentials(auth_code) st.session_state.session_state['google_creds'] = creds st.write(msg) if mode == "Manual URL": manual_url_mode() elif mode == "Bing Search": bing_search_mode() else: pdf_summarizer_mode() def manual_url_mode(): st.header("Manual URL Mode") url = st.text_input("Enter URL", placeholder="https://example.com") if st.button("Deep Search"): if url: async def run_deep_search(): async with DownloadManager( use_proxy=st.session_state.get('use_proxy', False), proxy=st.session_state.get('proxy', None) ) as dm: files = await dm.deep_search( url=url, custom_ext_list=st.session_state.get('custom_extensions', '').split(','), max_concurrency=st.session_state.get('max_concurrency', 200) ) st.session_state.session_state['discovered_files'] = files st.session_state.session_state['current_url'] = url if files: st.write(f"Found {len(files)} files:") for f in files: st.write(f"- {f['filename']} ({f['size']})") else: st.warning("No files found.") asyncio.run(run_deep_search()) def bing_search_mode(): st.header("Bing Search Mode") query = st.text_input("Enter search query") num_results = st.slider("Number of results", 1, 50, 5) if st.button("Search"): if query: async def run_search(): async with DownloadManager( use_proxy=st.session_state.get('use_proxy', False), proxy=st.session_state.get('proxy', None), query=query, num_results=num_results ) as dm: urls, info = await dm.search_bing() if urls: st.write("Search Results:") for i, (url, info) in enumerate(zip(urls, info), 1): st.write(f"{i}. {url}") st.write(f" Snippet: {info['snippet']}") else: st.warning("No results found.") asyncio.run(run_search()) def pdf_summarizer_mode(): st.header("PDF Summarizer") pdf_url = st.text_input("Enter PDF URL") if st.button("Summarize"): if pdf_url: summary = summarize_pdf_url(pdf_url) st.write("Summary:") st.write(summary) if __name__ == "__main__": main()