File size: 2,547 Bytes
3943768 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 |
import os
import shutil
import uuid
from urllib.parse import urlparse
import requests
def is_url_valid_and_alive(url, timeout=5):
try:
# Check if the URL is valid
result = urlparse(url)
if all([result.scheme, result.netloc]):
# Try to send a GET request to the URL
response = requests.get(url, timeout=timeout)
# If the status code is less than 400, consider it alive
return response.status_code < 400
else:
return False
except requests.exceptions.RequestException:
return False
def filename_is_url(filename):
if filename and (filename.startswith('http://') or filename.startswith('https://') or filename.startswith('www.')):
if is_url_valid_and_alive(filename):
return True
return False
def download_simple(url, dest=None, overwrite=False, verbose=False):
if dest is None:
dest = os.path.basename(url)
base_path = os.path.dirname(dest)
if base_path: # else local path
os.makedirs(base_path, exist_ok=True)
dest = os.path.join(base_path, os.path.basename(dest))
if os.path.isfile(dest):
if not overwrite:
if verbose:
print("Already have %s from url %s, delete file if invalid" % (dest, str(url)), flush=True)
return dest
else:
os.remove(dest)
if verbose:
print("BEGIN get url %s" % str(url), flush=True)
if url.startswith("file://"):
from requests_file import FileAdapter
s = requests.Session()
s.mount('file://', FileAdapter())
url_data = s.get(url, stream=True)
else:
url_data = requests.get(url, stream=True)
if verbose:
print("GOT url %s" % str(url), flush=True)
if url_data.status_code != requests.codes.ok:
msg = "Cannot get url %s, code: %s, reason: %s" % (
str(url),
str(url_data.status_code),
str(url_data.reason),
)
raise requests.exceptions.RequestException(msg)
url_data.raw.decode_content = True
uuid_tmp = str(uuid.uuid4())[:6]
dest_tmp = dest + "_dl_" + uuid_tmp + ".tmp"
# Sizes in bytes.
block_size = 1024
with open(dest_tmp, "wb") as file:
for data in url_data.iter_content(block_size):
file.write(data)
try:
shutil.move(dest_tmp, dest)
except (shutil.Error, FileExistsError):
pass
if verbose:
print("DONE url %s" % str(url), flush=True)
return dest
|