import os import re import sys import json import time import shutil import tempfile import warnings from typing import Optional, Union, IO import requests from urllib.parse import urlparse, unquote from tqdm import tqdm CHUNK_SIZE = 512 * 1024 HOME = os.path.expanduser("~") def indent(text: str, prefix: str): """Indent each non-empty line of text with the given prefix.""" return "".join( (prefix + line if line.strip() else line) for line in text.splitlines(True) ) class FileURLRetrievalError(Exception): """Custom exception for issues retrieving file URLs.""" def _extract_download_url_from_confirmation(contents: str, url_origin: str): """Extract the download URL from a Google Drive confirmation page.""" patterns = [ r'href="(\/uc\?export=download[^"]+)', r'href="/open\?id=([^"]+)"', r'"downloadUrl":"([^"]+)', ] for pattern in patterns: match = re.search(pattern, contents) if match: url = match.group(1) if pattern == r'href="/open\?id=([^"]+)"': uuid_match = re.search( r'(.*)

', contents) if error_match: error = error_match.group(1) raise FileURLRetrievalError(error) raise FileURLRetrievalError( "Cannot retrieve the public link of the file. " "You may need to change the permission to " "'Anyone with the link', or have had many accesses." ) def _create_session( proxy: Optional[str] = None, use_cookies: bool = True, return_cookies_file: bool = False, ): """Create a requests session with optional proxy and cookie handling.""" sess = requests.session() sess.headers.update( {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6)"} ) if proxy: sess.proxies = {"http": proxy, "https": proxy} cookies_file = os.path.join(HOME, ".cache/gdown/cookies.json") if os.path.exists(cookies_file) and use_cookies: try: with open(cookies_file) as f: cookies = json.load(f) for k, v in cookies: sess.cookies[k] = v except json.JSONDecodeError: warnings.warn("Corrupted Cookies file") return (sess, cookies_file) if return_cookies_file else sess def download( output: Optional[str] = None, quiet: bool = False, proxy: Optional[str] = None, speed: Optional[float] = None, use_cookies: bool = True, verify: Union[bool, str] = True, id: Optional[str] = None, fuzzy: bool = True, resume: bool = False, format: Optional[str] = None, url: Optional[str] = None, ): """Download a file from a URL, supporting Google Drive links. Args: output: Output filepath. Default is basename of URL. quiet: Suppress terminal output. proxy: HTTP/HTTPS proxy. speed: Download speed limit (bytes per second). use_cookies: Flag to use cookies. verify: Verify TLS certificates. id: Google Drive's file ID. fuzzy: Fuzzy Google Drive ID extraction. resume: Resume download from a tmp file. format: Format for Google Docs/Sheets/Slides. url: URL to download from. Returns: Output filename, or None on error. """ if not (id is None) ^ (url is None): raise ValueError("Either url or id has to be specified") if id is not None: url = f"https://drive.google.com/uc?id={id}" url_origin = url sess, cookies_file = _create_session( proxy=proxy, use_cookies=use_cookies, return_cookies_file=True ) while True: res = sess.get(url, stream=True, verify=verify) res.raise_for_status() if url == url_origin and res.status_code == 500: url = f"https://drive.google.com/open?id={id}" continue if res.headers.get("Content-Type", "").startswith("text/html"): title_match = re.search("(.+)", res.text) if title_match: title = title_match.group(1) if title.endswith(" - Google Docs"): url = f"https://docs.google.com/document/d/{id}/export?format={'docx' if format is None else format}" continue if title.endswith(" - Google Sheets"): url = f"https://docs.google.com/spreadsheets/d/{id}/export?format={'xlsx' if format is None else format}" continue if title.endswith(" - Google Slides"): url = f"https://docs.google.com/presentation/d/{id}/export?format={'pptx' if format is None else format}" continue if ( "Content-Disposition" in res.headers and res.headers["Content-Disposition"].endswith("pptx") and format not in (None, "pptx") ): url = f"https://docs.google.com/presentation/d/{id}/export?format={'pptx' if format is None else format}" continue if use_cookies: os.makedirs(os.path.dirname(cookies_file), exist_ok=True) cookies = [ (k, v) for k, v in sess.cookies.items() if not k.startswith("download_warning_") ] with open(cookies_file, "w") as f: json.dump(cookies, f, indent=2) if "Content-Disposition" in res.headers: break parsed_url = urlparse(url) is_gdrive = parsed_url.hostname in ("drive.google.com", "docs.google.com") is_download_link = parsed_url.path.endswith("/uc") if not (is_gdrive and is_download_link and fuzzy): break try: url = _extract_download_url_from_confirmation(res.text, url_origin) except FileURLRetrievalError as e: raise FileURLRetrievalError(e) content_disposition = res.headers.get("Content-Disposition", "") filename_match = re.search( r"filename\*=UTF-8''(.*)", content_disposition ) or re.search(r'filename=["\']?(.*?)["\']?$', content_disposition) filename_from_url = ( unquote(filename_match.group(1)) if filename_match else os.path.basename(url) ) download_path = output or filename_from_url if isinstance(download_path, str) and download_path.endswith(os.path.sep): os.makedirs(download_path, exist_ok=True) download_path = os.path.join(download_path, filename_from_url) temp_dir = os.path.dirname(download_path) or "." prefix = os.path.basename(download_path) if isinstance(download_path, str): existing_tmp_files = [ os.path.join(temp_dir, file) for file in os.listdir(temp_dir) if file.startswith(prefix) ] if resume and existing_tmp_files: if len(existing_tmp_files) > 1: print( "There are multiple temporary files to resume:", file=sys.stderr, ) for file in existing_tmp_files: print(f"\t{file}", file=sys.stderr) print( "Please remove them except one to resume downloading.", file=sys.stderr, ) return None temp_file_path = existing_tmp_files[0] else: resume = False temp_file_path = tempfile.mktemp( suffix=tempfile.template, prefix=prefix, dir=temp_dir ) try: file_obj: IO = open(temp_file_path, "ab") except Exception as e: print( f"Could not open the temporary file {temp_file_path}: {e}", file=sys.stderr, ) return None else: temp_file_path = None file_obj = download_path if temp_file_path is not None and file_obj.tell() != 0: headers = {"Range": f"bytes={file_obj.tell()}-"} res = sess.get(url, headers=headers, stream=True, verify=verify) res.raise_for_status() try: total = int(res.headers.get("Content-Length", 0)) if total > 0: if not quiet: pbar = tqdm( total=total, unit="B", unit_scale=True, desc=filename_from_url ) else: if not quiet: pbar = tqdm(unit="B", unit_scale=True, desc=filename_from_url) t_start = time.time() for chunk in res.iter_content(chunk_size=CHUNK_SIZE): file_obj.write(chunk) if not quiet: pbar.update(len(chunk)) if speed is not None: elapsed_time_expected = 1.0 * pbar.n / speed elapsed_time = time.time() - t_start if elapsed_time < elapsed_time_expected: time.sleep(elapsed_time_expected - elapsed_time) if not quiet: pbar.close() if temp_file_path: file_obj.close() shutil.move(temp_file_path, download_path) finally: sess.close() return download_path