Upload 7 files

README.md

@@ -4,7 +4,7 @@ emoji: 🤗
 colorFrom: yellow
 colorTo: blue
 sdk: gradio
-sdk_version: 5.0.1
+sdk_version: 5.1.0
 app_file: app.py
 pinned: false
 license: mit

app.py

@@ -1,6 +1,6 @@
 import gradio as gr
 from civitai_to_hf import (search_civitai, download_civitai, select_civitai_item, add_civitai_item, get_civitai_tag, select_civitai_all_item,
-                           CIVITAI_TYPE, CIVITAI_BASEMODEL, CIVITAI_SORT, CIVITAI_PERIOD)
+                           update_civitai_selection, CIVITAI_TYPE, CIVITAI_BASEMODEL, CIVITAI_SORT, CIVITAI_PERIOD)
 
 css = """
 .title { font-size: 3em; align-items: center; text-align: center; }
@@ -32,6 +32,8 @@ with gr.Blocks(theme="NoCrypt/miku@>=1.2.2", fill_width=True, css=css, delete_ca
                     search_civitai_json = gr.JSON(value={}, visible=False)
                 with gr.Row():
                     with gr.Column(scale=9):
+                        with gr.Accordion("Select from Gallery", open=False):
+                            search_civitai_gallery = gr.Gallery([], label="Results", allow_preview=False, columns=5, elem_id="gallery", show_share_button=False, interactive=False)
                         search_civitai_result = gr.Dropdown(label="Search Results", choices=[("", "")], value=[],
                                                             allow_custom_value=True, visible=True, multiselect=True)
                     with gr.Column(scale=1):
@@ -67,14 +69,15 @@ with gr.Blocks(theme="NoCrypt/miku@>=1.2.2", fill_width=True, css=css, delete_ca
         triggers=[search_civitai_submit.click, search_civitai_query.submit, search_civitai_user.submit],
         fn=search_civitai,
         inputs=[search_civitai_query, search_civitai_type, search_civitai_basemodel, search_civitai_sort,
-                search_civitai_period, search_civitai_tag, search_civitai_user, search_civitai_limit, search_civitai_page],
-        outputs=[search_civitai_result, search_civitai_desc, search_civitai_submit, search_civitai_query],
+                search_civitai_period, search_civitai_tag, search_civitai_user, search_civitai_limit, search_civitai_page, search_civitai_gallery],
+        outputs=[search_civitai_result, search_civitai_desc, search_civitai_submit, search_civitai_query, search_civitai_gallery],
         queue=True,
         show_api=False,
     )
     search_civitai_result.change(select_civitai_item, [search_civitai_result], [search_civitai_desc, search_civitai_json], queue=False, show_api=False)
     search_civitai_add.click(add_civitai_item, [search_civitai_result, dl_url], [dl_url], queue=False, show_api=False)
     search_civitai_select_all.click(select_civitai_all_item, [search_civitai_select_all], [search_civitai_select_all, search_civitai_result], queue=False, show_api=False)
+    search_civitai_gallery.select(update_civitai_selection, [search_civitai_result], [search_civitai_result], queue=False, show_api=False)
 
 demo.queue()
 demo.launch()

civitai_to_hf.py

@@ -166,7 +166,9 @@ def search_on_civitai(query: str, types: list[str], allow_model: list[str] = [],
                 item['dl_url'] = model['downloadUrl']
                 item['md'] = ""
                 if 'images' in model.keys() and len(model["images"]) != 0:
+                    item['img_url'] = model["images"][0]["url"]
                     item['md'] += f'<img src="{model["images"][0]["url"]}#float" alt="thumbnail" width="150" height="240"><br>'
+                else: item['img_url'] = "/home/user/app/null.png"
                 item['md'] += f'''Model URL: [https://civitai.com/models/{j["id"]}](https://civitai.com/models/{j["id"]})<br>Model Name: {item["name"]}<br>
                     Creator: {item["creator"]}<br>Tags: {", ".join(item["tags"])}<br>Base Model: {item["base_model"]}<br>Description: {item["description"]}'''
                 items.append(item)
@@ -175,30 +177,34 @@ def search_on_civitai(query: str, types: list[str], allow_model: list[str] = [],
 
 civitai_last_results = {}
 civitai_last_choices = [("", "")]
+civitai_last_gallery = []
 
-
-def search_civitai(query, types, base_model=[], sort=CIVITAI_SORT[0], period=CIVITAI_PERIOD[0], tag="", user="", limit=100, page=1):
-    global civitai_last_results, civitai_last_choices
+def search_civitai(query, types, base_model=[], sort=CIVITAI_SORT[0], period=CIVITAI_PERIOD[0], tag="", user="", limit=100, page=1, gallery=[]):
+    global civitai_last_results, civitai_last_choices, civitai_last_gallery
     civitai_last_choices = [("", "")]
+    civitai_last_gallery = []
     civitai_last_results = {}
     items = search_on_civitai(query, types, base_model, int(limit), sort, period, tag, user, int(page))
     if not items: return gr.update(choices=[("", "")], value=[], visible=True),\
-          gr.update(value="", visible=False), gr.update(visible=True), gr.update(visible=True)
+          gr.update(value="", visible=False), gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)
     choices = []
+    gallery = []
     for item in items:
         base_model_name = "Pony🐴" if item['base_model'] == "Pony" else item['base_model']
         name = f"{item['name']} (for {base_model_name} / By: {item['creator']} / Tags: {', '.join(item['tags'])})"
         value = item['dl_url']
         choices.append((name, value))
+        gallery.append((item['img_url'], name))
         civitai_last_results[value] = item
     if not choices: return gr.update(choices=[("", "")], value=[], visible=True),\
-          gr.update(value="", visible=False), gr.update(visible=True), gr.update(visible=True)
+          gr.update(value="", visible=False), gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)
     #result = civitai_last_results.get(choices[0][1], "None") # for debug
     #md = result['md'] if result else "" # for debug
     md = ""
     civitai_last_choices = choices
+    civitai_last_gallery = gallery
     return gr.update(choices=choices, value=[], visible=True), gr.update(value=md, visible=True),\
-          gr.update(visible=True), gr.update(visible=True)
+          gr.update(visible=True), gr.update(visible=True), gr.update(value=gallery)
 
 
 def get_civitai_json(dl_url: str, is_html: bool=False, image_baseurl: str=""):
@@ -291,3 +297,13 @@ def select_civitai_all_item(button_name: str):
     selected = [t[1] for t in civitai_last_choices if t[1] != ""] if button_name == "Select All" else []
     new_button_name = "Select All" if button_name == "Deselect All" else "Deselect All"
     return gr.update(value=new_button_name), gr.update(value=selected, choices=civitai_last_choices)
+
+
+def update_civitai_selection(evt: gr.SelectData, value: list[str]):
+    try:
+        selected_index = evt.index
+        selected = list_uniq([v for v in value if v != ""] + [civitai_last_choices[selected_index][1]])
+        return gr.update(value=selected)
+    except Exception:
+        return gr.update(visible=True)
+

utils.py

@@ -1,5 +1,5 @@
 import gradio as gr
-from huggingface_hub import HfApi, HfFolder, hf_hub_download
+from huggingface_hub import HfApi, HfFolder, hf_hub_download, snapshot_download
 import os
 from pathlib import Path
 import shutil
@@ -64,7 +64,7 @@ def get_model_type(repo_id: str):
 
 
 def list_uniq(l):
-        return sorted(set(l), key=l.index)
+    return sorted(set(l), key=l.index)
 
 
 def list_sub(a, b):
@@ -92,14 +92,16 @@ def download_hf_file(directory, url, progress=gr.Progress(track_tqdm=True)):
     hf_token = get_token()
     repo_id, filename, subfolder, repo_type = split_hf_url(url)
     try:
-        if subfolder is not None: hf_hub_download(repo_id=repo_id, filename=filename, subfolder=subfolder, repo_type=repo_type, local_dir=directory, token=hf_token)
-        else: hf_hub_download(repo_id=repo_id, filename=filename, repo_type=repo_type, local_dir=directory, token=hf_token)
+        print(f"Downloading {url} to {directory}")
+        if subfolder is not None: path = hf_hub_download(repo_id=repo_id, filename=filename, subfolder=subfolder, repo_type=repo_type, local_dir=directory, token=hf_token)
+        else: path = hf_hub_download(repo_id=repo_id, filename=filename, repo_type=repo_type, local_dir=directory, token=hf_token)
+        return path
     except Exception as e:
         print(f"Failed to download: {e}")
+        return None
 
 
 def download_thing(directory, url, civitai_api_key="", progress=gr.Progress(track_tqdm=True)): # requires aria2, gdown
-    hf_token = get_token()
     url = url.strip()
     if "drive.google.com" in url:
         original_dir = os.getcwd()
@@ -108,14 +110,8 @@ def download_thing(directory, url, civitai_api_key="", progress=gr.Progress(trac
         os.chdir(original_dir)
     elif "huggingface.co" in url:
         url = url.replace("?download=true", "")
-        if "/blob/" in url:
-            url = url.replace("/blob/", "/resolve/")
-        #user_header = f'"Authorization: Bearer {hf_token}"'
-        if hf_token:
-            download_hf_file(directory, url)
-            #os.system(f"aria2c --console-log-level=error --summary-interval=10 --header={user_header} -c -x 16 -k 1M -s 16 {url} -d {directory}  -o {url.split('/')[-1]}")
-        else:
-            os.system(f"aria2c --optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 {url} -d {directory}  -o {url.split('/')[-1]}")
+        if "/blob/" in url: url = url.replace("/blob/", "/resolve/")
+        download_hf_file(directory, url)
     elif "civitai.com" in url:
         if "?" in url:
             url = url.split("?")[0]
@@ -128,14 +124,13 @@ def download_thing(directory, url, civitai_api_key="", progress=gr.Progress(trac
         os.system(f"aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {directory} {url}")
 
 
-def get_local_model_list(dir_path):
-    model_list = []
-    valid_extensions = ('.safetensors')
+def get_local_file_list(dir_path):
+    file_list = []
     for file in Path(dir_path).glob("**/*.*"):
-        if file.is_file() and file.suffix in valid_extensions:
+        if file.is_file():
             file_path = str(file)
-            model_list.append(file_path)
-    return model_list
+            file_list.append(file_path)
+    return file_list
 
 
 def get_download_file(temp_dir, url, civitai_key, progress=gr.Progress(track_tqdm=True)):
@@ -150,16 +145,48 @@ def get_download_file(temp_dir, url, civitai_key, progress=gr.Progress(track_tqd
         new_file = f"{temp_dir}/{url.split('/')[-1]}"
     else:
         print(f"Start downloading: {url}")
-        before = get_local_model_list(temp_dir)
+        before = get_local_file_list(temp_dir)
         try:
             download_thing(temp_dir, url.strip(), civitai_key)
         except Exception:
             print(f"Download failed: {url}")
             return ""
-        after = get_local_model_list(temp_dir)
+        after = get_local_file_list(temp_dir)
         new_file = list_sub(after, before)[0] if list_sub(after, before) else ""
     if not new_file:
         print(f"Download failed: {url}")
         return ""
     print(f"Download completed: {url}")
     return new_file
+
+
+# https://huggingface.co/docs/huggingface_hub/v0.25.1/en/package_reference/file_download#huggingface_hub.snapshot_download
+def download_repo(repo_id, dir_path, progress=gr.Progress(track_tqdm=True)):
+    hf_token = get_token()
+    try:
+        snapshot_download(repo_id=repo_id, local_dir=dir_path, token=hf_token, allow_patterns=["*.safetensors", "*.bin"],
+                          ignore_patterns=["*.fp16.*", "/*.safetensors", "/*.bin"], force_download=True)
+        return True
+    except Exception as e:
+        print(f"Error: Failed to download {repo_id}. {e}")
+        gr.Warning(f"Error: Failed to download {repo_id}. {e}")
+        return False
+
+
+def upload_repo(new_repo_id, dir_path, is_private, progress=gr.Progress(track_tqdm=True)):
+    hf_token = get_token()
+    api = HfApi(token=hf_token)
+    try:
+        progress(0, desc="Start uploading...")
+        api.create_repo(repo_id=new_repo_id, token=hf_token, private=is_private, exist_ok=True)
+        for path in Path(dir_path).glob("*"):
+            if path.is_dir():
+                api.upload_folder(repo_id=new_repo_id, folder_path=str(path), path_in_repo=path.name, token=hf_token)
+            elif path.is_file():
+                api.upload_file(repo_id=new_repo_id, path_or_fileobj=str(path), path_in_repo=path.name, token=hf_token)
+        progress(1, desc="Uploaded.")
+        url = f"https://huggingface.co/{new_repo_id}"
+    except Exception as e:
+        print(f"Error: Failed to upload to {new_repo_id}. {e}")
+        return ""
+    return url