Update main.py
Browse files
main.py
CHANGED
@@ -155,14 +155,35 @@ class CivitAICrawler:
|
|
155 |
logger.error("Failed after %d attempts: %s", max_retries, enc_file_path)
|
156 |
raise
|
157 |
|
158 |
-
def upload_folder_encrypted(self, folder_path: str, repo_id: Optional[str] = None, path_in_repo: str = ""):
|
159 |
-
|
|
|
160 |
repo_id = self.repo_ids['current']
|
|
|
|
|
161 |
self.encrypt_with_rclone(folder_path)
|
162 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
163 |
if os.path.isdir(self.config.ENCRYPTED_DIR):
|
164 |
shutil.rmtree(self.config.ENCRYPTED_DIR, ignore_errors=True)
|
165 |
|
|
|
|
|
|
|
166 |
def upload_file_encrypted_one_by_one(self, file_path: str, repo_id: Optional[str] = None, path_in_repo: str = ""):
|
167 |
"""古いバージョン用: ファイルを暗号化アップロード → ローカル削除。"""
|
168 |
if not repo_id:
|
@@ -409,11 +430,11 @@ class CivitAICrawler:
|
|
409 |
encrypted_top_name = self.upload_folder_encrypted(folder)
|
410 |
shutil.rmtree(folder, ignore_errors=True)
|
411 |
|
412 |
-
#
|
413 |
-
|
414 |
-
|
415 |
-
|
416 |
-
self.append_model_list(model_id,
|
417 |
|
418 |
except Exception as e:
|
419 |
logger.error("Unexpected error in process_model(%s): %s", model_url, e)
|
|
|
155 |
logger.error("Failed after %d attempts: %s", max_retries, enc_file_path)
|
156 |
raise
|
157 |
|
158 |
+
def upload_folder_encrypted(self, folder_path: str, repo_id: Optional[str] = None, path_in_repo: str = "") -> str:
|
159 |
+
"""フォルダを暗号化してアップロードし、暗号化後のトップフォルダ名を返す。"""
|
160 |
+
if repo_id is None:
|
161 |
repo_id = self.repo_ids['current']
|
162 |
+
|
163 |
+
# rclone でローカル -> cryptLocal:
|
164 |
self.encrypt_with_rclone(folder_path)
|
165 |
+
|
166 |
+
# 暗号化先ディレクトリ直下を見て、暗号化後のトップレベルフォルダ名を取得
|
167 |
+
# (rclone の directory_name_encryption=standard 等ならランダム文字列になっているはず)
|
168 |
+
top_levels = [
|
169 |
+
d for d in os.listdir(self.config.ENCRYPTED_DIR)
|
170 |
+
if os.path.isdir(os.path.join(self.config.ENCRYPTED_DIR, d))
|
171 |
+
]
|
172 |
+
if not top_levels:
|
173 |
+
raise RuntimeError("No top-level folder found after encryption.")
|
174 |
+
# もし複数あったら最初の1つだけ使う
|
175 |
+
encrypted_top_name = top_levels[0]
|
176 |
+
|
177 |
+
# Upload
|
178 |
+
self.upload_encrypted_files(repo_id, path_in_repo)
|
179 |
+
|
180 |
+
# 後片付け
|
181 |
if os.path.isdir(self.config.ENCRYPTED_DIR):
|
182 |
shutil.rmtree(self.config.ENCRYPTED_DIR, ignore_errors=True)
|
183 |
|
184 |
+
# 呼び出し元がURLを作れるよう返す
|
185 |
+
return encrypted_top_name
|
186 |
+
|
187 |
def upload_file_encrypted_one_by_one(self, file_path: str, repo_id: Optional[str] = None, path_in_repo: str = ""):
|
188 |
"""古いバージョン用: ファイルを暗号化アップロード → ローカル削除。"""
|
189 |
if not repo_id:
|
|
|
430 |
encrypted_top_name = self.upload_folder_encrypted(folder)
|
431 |
shutil.rmtree(folder, ignore_errors=True)
|
432 |
|
433 |
+
# ここで、暗号化後のトップフォルダ名をURLに含める
|
434 |
+
model_hf_url = f"https://huggingface.co/{self.repo_ids['current']}/tree/main/{encrypted_top_name}"
|
435 |
+
|
436 |
+
# あるいは、さらにサブフォルダ構造があるなら適宜URLを調整
|
437 |
+
self.append_model_list(model_id, model_hf_url)
|
438 |
|
439 |
except Exception as e:
|
440 |
logger.error("Unexpected error in process_model(%s): %s", model_url, e)
|