Upload 7 files

app.py

@@ -62,13 +62,14 @@ with gr.Blocks(theme="NoCrypt/miku@>=1.2.2", fill_width=True, css=css, delete_ca
         run_button = gr.Button(value="Download and Upload", variant="primary")
         uploaded_urls = gr.CheckboxGroup(visible=False, choices=[], value=None) # hidden
         urls_md = gr.Markdown("<br><br>", elem_classes="result")
+        urls_remain = gr.Textbox("Remaining URLs", value="", show_copy_button=True, visible=False)
     gr.DuplicateButton(value="Duplicate Space")
 
     gr.on(
         triggers=[run_button.click],
         fn=download_civitai,
         inputs=[dl_url, civitai_key, hf_token, uploaded_urls, newrepo_id, newrepo_type, is_private, is_info],
-        outputs=[uploaded_urls, urls_md],
+        outputs=[uploaded_urls, urls_md, urls_remain],
     )
     gr.on(
         triggers=[search_civitai_submit.click, search_civitai_query.submit, search_civitai_user.submit],

civitai_to_hf.py

@@ -6,7 +6,7 @@ import gc
 import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util import Retry
-from utils import (get_token, set_token, is_repo_name, is_repo_exists, get_user_agent, get_download_file,
+from utils import (get_token, set_token, is_repo_exists, get_user_agent, get_download_file,
                    list_uniq, duplicate_hf_repo, HF_SUBFOLDER_NAME, get_state, set_state)
 import re
 from PIL import Image
@@ -60,7 +60,7 @@ def upload_safetensors_to_repo(filename, repo_id, repo_type, is_private, progres
         gr.Warning(f"Error: Failed to upload to {repo_id}. {e}")
         return None
     finally:
-        Path(filename).unlink()
+        if Path(filename).exists(): Path(filename).unlink()
     return url
 
 
@@ -121,23 +121,28 @@ def download_civitai(dl_url, civitai_key, hf_token, urls,
     if not get_token() or not civitai_key: raise gr.Error("HF write token and Civitai API key is required.")
     if not urls: urls = []
     dl_urls = parse_urls(dl_url)
-    for u in dl_urls:
-        file = download_file(u, civitai_key)
-        if not Path(file).exists() or not Path(file).is_file(): continue
-        url = upload_safetensors_to_repo(file, newrepo_id, repo_type, is_private)
-        if url:
-            urls.append(url)
-            if is_info: upload_info_to_repo(u, file, newrepo_id, repo_type, is_private)
-    dp_repos = parse_repos(dl_url)
-    for r in dp_repos:
-        url = duplicate_hf_repo(r, newrepo_id, "model", repo_type, is_private, HF_SUBFOLDER_NAME[1])
-        if url: urls.append(url)
-    progress(1, desc="Processing...")
-    md = f'### Your repo: [{newrepo_id}]({"https://huggingface.co/datasets/" if repo_type == "dataset" else "https://huggingface.co/"}{newrepo_id})\n'
-    for u in urls:
-        md += f"- Uploaded [{str(u)}]({str(u)})\n"
-    gc.collect()
-    return gr.update(value=urls, choices=urls), gr.update(value=md)
+    remain_urls = dl_urls.copy()
+    try:
+        md = f'### Your repo: [{newrepo_id}]({"https://huggingface.co/datasets/" if repo_type == "dataset" else "https://huggingface.co/"}{newrepo_id})\n'
+        for u in dl_urls:
+            file = download_file(u, civitai_key)
+            if not Path(file).exists() or not Path(file).is_file(): continue
+            url = upload_safetensors_to_repo(file, newrepo_id, repo_type, is_private)
+            if url:
+                if is_info: upload_info_to_repo(u, file, newrepo_id, repo_type, is_private)
+                urls.append(url)
+                remain_urls.remove(u)
+                md += f"- Uploaded [{str(u)}]({str(u)})\n"
+        dp_repos = parse_repos(dl_url)
+        for r in dp_repos:
+            url = duplicate_hf_repo(r, newrepo_id, "model", repo_type, is_private, HF_SUBFOLDER_NAME[1])
+            if url: urls.append(url)
+        return gr.update(value=urls, choices=urls), gr.update(value=md), gr.update(value="\n".join(remain_urls), visible=False)
+    except Exception as e:
+        gr.Info(f"Error occured: {e}")
+        return gr.update(value=urls, choices=urls), gr.update(value=md), gr.update(value="\n".join(remain_urls), visible=True)
+    finally:
+        gc.collect()
 
 
 CIVITAI_TYPE = ["Checkpoint", "TextualInversion", "Hypernetwork", "AestheticGradient", "LORA", "LoCon", "DoRA",
@@ -291,7 +296,6 @@ def select_civitai_item(results: list[str], state: dict):
     if "http" not in "".join(results) or len(results) == 0: return gr.update(value="", visible=True), gr.update(value=json, visible=False)
     result = get_state(state, "civitai_last_results")
     md = result.get('md', "") if result and isinstance(result, dict) else ""
-    #json, html, image = get_civitai_json(results[-1], True) # for debugging
     return gr.update(value=md, visible=True), gr.update(value=json, visible=False)
 
 

utils.py

@@ -125,26 +125,29 @@ def download_hf_file(directory, url, progress=gr.Progress(track_tqdm=True)):
 
 
 def download_thing(directory, url, civitai_api_key="", progress=gr.Progress(track_tqdm=True)): # requires aria2, gdown
-    url = url.strip()
-    if "drive.google.com" in url:
-        original_dir = os.getcwd()
-        os.chdir(directory)
-        os.system(f"gdown --fuzzy {url}")
-        os.chdir(original_dir)
-    elif "huggingface.co" in url:
-        url = url.replace("?download=true", "")
-        if "/blob/" in url: url = url.replace("/blob/", "/resolve/")
-        download_hf_file(directory, url)
-    elif "civitai.com" in url:
-        if "?" in url:
-            url = url.split("?")[0]
-        if civitai_api_key:
-            url = url + f"?token={civitai_api_key}"
-            os.system(f"aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {directory} {url}")
+    try:
+        url = url.strip()
+        if "drive.google.com" in url:
+            original_dir = os.getcwd()
+            os.chdir(directory)
+            os.system(f"gdown --fuzzy {url}")
+            os.chdir(original_dir)
+        elif "huggingface.co" in url:
+            url = url.replace("?download=true", "")
+            if "/blob/" in url: url = url.replace("/blob/", "/resolve/")
+            download_hf_file(directory, url)
+        elif "civitai.com" in url:
+            if "?" in url:
+                url = url.split("?")[0]
+            if civitai_api_key:
+                url = url + f"?token={civitai_api_key}"
+                os.system(f"aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {directory} {url}")
+            else:
+                print("You need an API key to download Civitai models.")
         else:
-            print("You need an API key to download Civitai models.")
-    else:
-        os.system(f"aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {directory} {url}")
+            os.system(f"aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {directory} {url}")
+    except Exception as e:
+        print(f"Failed to download: {e}")
 
 
 def get_local_file_list(dir_path):
@@ -157,30 +160,30 @@ def get_local_file_list(dir_path):
 
 
 def get_download_file(temp_dir, url, civitai_key, progress=gr.Progress(track_tqdm=True)):
-    if not "http" in url and is_repo_name(url) and not Path(url).exists():
-        print(f"Use HF Repo: {url}")
-        new_file = url
-    elif not "http" in url and Path(url).exists():
-        print(f"Use local file: {url}")
-        new_file = url
-    elif Path(f"{temp_dir}/{url.split('/')[-1]}").exists():
-        print(f"File to download alreday exists: {url}")
-        new_file = f"{temp_dir}/{url.split('/')[-1]}"
-    else:
-        print(f"Start downloading: {url}")
-        before = get_local_file_list(temp_dir)
-        try:
+    try:
+        if not "http" in url and is_repo_name(url) and not Path(url).exists():
+            print(f"Use HF Repo: {url}")
+            new_file = url
+        elif not "http" in url and Path(url).exists():
+            print(f"Use local file: {url}")
+            new_file = url
+        elif Path(f"{temp_dir}/{url.split('/')[-1]}").exists():
+            print(f"File to download alreday exists: {url}")
+            new_file = f"{temp_dir}/{url.split('/')[-1]}"
+        else:
+            print(f"Start downloading: {url}")
+            before = get_local_file_list(temp_dir)
             download_thing(temp_dir, url.strip(), civitai_key)
-        except Exception:
+            after = get_local_file_list(temp_dir)
+            new_file = list_sub(after, before)[0] if list_sub(after, before) else ""
+        if not new_file:
             print(f"Download failed: {url}")
             return ""
-        after = get_local_file_list(temp_dir)
-        new_file = list_sub(after, before)[0] if list_sub(after, before) else ""
-    if not new_file:
+        print(f"Download completed: {url}")
+        return new_file
+    except Exception:
         print(f"Download failed: {url}")
         return ""
-    print(f"Download completed: {url}")
-    return new_file
 
 
 def download_repo(repo_id: str, dir_path: str, progress=gr.Progress(track_tqdm=True)): # for diffusers repo