Update main.py

main.py

@@ -1,627 +1,97 @@
-import asyncio
-import datetime
-import json
-import logging
-import os
-import re
-import shutil
-import subprocess
-import time
-import uuid
-from typing import Optional
-
-import requests
-from bs4 import BeautifulSoup
-from fake_useragent import UserAgent
-from fastapi import FastAPI
-from huggingface_hub import HfApi, hf_hub_download, login
-
-# ロギングの設定
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-class Config:
-    """設定用のクラス"""
-    HUGGINGFACE_API_KEY = os.environ["HUGGINGFACE_API_KEY"]
-    CIVITAI_API_TOKEN = os.environ["CIVITAI_API_TOKEN"]
-    LOG_FILE = "civitai_backup.log"
-    LIST_FILE = "model_list.log"
-    REPO_IDS = {
-        "log": "ttttdiva/CivitAI_log_test",
-        "model_list": "ttttdiva/CivitAI_model_info_test",
-        "current": ""
-    }
-    URLS = {
-        "latest": "https://civitai.com/api/v1/models?sort=Newest",
-        "modelPage": "https://civitai.com/models/",
-        "modelId": "https://civitai.com/api/v1/models/",
-        "modelVersionId": "https://civitai.com/api/v1/model-versions/",
-        "hash": "https://civitai.com/api/v1/model-versions/by-hash/"
-    }
-    JST = datetime.timezone(datetime.timedelta(hours=9))
-    UA = UserAgent()
-    HEADERS = {
-        'Authorization': f'Bearer {CIVITAI_API_TOKEN}',
-        'User-Agent': UA.random,
-        "Content-Type": "application/json"
-    }
-
-
-class CivitAICrawler:
-    """CivitAIからモデルをダウンロードし、Hugging Faceにアップロードするクラス"""
-
-    def __init__(self, config: Config):
-        import base64
-
-        rclone_conf_base64 = os.environ.get("RCLONE_CONF_BASE64")
-        if rclone_conf_base64:
-            # カレントディレクトリ配下に .rclone_config ディレクトリを作成
-            config_dir = os.path.join(os.getcwd(), ".rclone_config")
-            os.makedirs(config_dir, exist_ok=True)
-
-            conf_path = os.path.join(config_dir, "rclone.conf")
-            with open(conf_path, "wb") as f:
-                f.write(base64.b64decode(rclone_conf_base64))
-
-            # rclone がここを参照するように設定
-            os.environ["RCLONE_CONFIG"] = conf_path
-            logger.info(f"[INFO] Created rclone.conf at {conf_path}")
-        else:
-            logger.warning("[WARN] RCLONE_CONF_BASE64 not found; rclone may fail.")
-
-        self.config = config
-        self.api = HfApi()
-        self.app = FastAPI()
-        self.repo_ids = self.config.REPO_IDS.copy()
-        self.jst = self.config.JST
-        self.setup_routes()
-
-    def setup_routes(self):
-        """FastAPIのルーティングを設定する。"""
-        @self.app.get("/")
-        def read_root():
-            now = str(datetime.datetime.now(self.jst))
-            description = f"""
-            CivitAIを定期的に周回し新規モデルを {self.repo_ids['current']} にバックアップするspaceです。
-            モデル一覧は https://huggingface.co/{self.repo_ids['model_list']}/blob/main/model_list.log を参照してください。
-            Status: {now} + currently running :D
-            """
-            return description
-
-        @self.app.on_event("startup")
-        async def startup_event():
-            asyncio.create_task(self.crawl())
-
-    @staticmethod
-    def get_filename_from_cd(content_disposition: Optional[str], default_name: str) -> str:
-        """Content-Dispositionヘッダーからファイル名を取得する。"""
-        if content_disposition:
-            parts = content_disposition.split(';')
-            for part in parts:
-                if "filename=" in part:
-                    return part.split("=")[1].strip().strip('"')
-        return default_name
-
-    def download_file(self, url: str, destination_folder: str, default_name: str) -> Optional[str]:
-        """指定されたURLからファイルをダウンロードし、指定されたフォルダに保存する。"""
-        try:
-            response = requests.get(url, headers=self.config.HEADERS, stream=True)
-            response.raise_for_status()
-        except requests.RequestException as e:
-            logger.error(f"Failed to download file from {url}: {e}")
-            return None
-
-        filename = self.get_filename_from_cd(response.headers.get('content-disposition'), default_name)
-        file_path = os.path.join(destination_folder, filename)
-
-        # ダウンロードとファイル保存処理
-        with open(file_path, 'wb') as file:
-            for chunk in response.iter_content(chunk_size=8192):
-                file.write(chunk)
-        logger.info(f"Downloaded: {file_path}")
-        return file_path
-
-    def get_model_info(self, model_id: str) -> dict:
-        """モデルの情報を取得する。"""
-        try:
-            response = requests.get(self.config.URLS["modelId"] + str(model_id), headers=self.config.HEADERS)
-            response.raise_for_status()
-            return response.json()
-        except requests.RequestException as e:
-            logger.error(f"Failed to retrieve model info for ID {model_id}: {e}")
-            return {}
-
-    def download_images(self, model_versions: list, folder: str):
-        """画像を images フォルダにまとめてダウンロードする."""
-        images_folder = os.path.join(folder, "images")
-        os.makedirs(images_folder, exist_ok=True)
-        images = []
-        for version in model_versions:
-            for img in version.get("images", []):
-                images.append(img["url"])
-        for image_url in images:
-            image_name = os.path.basename(image_url)
-            local_path = os.path.join(images_folder, image_name)
-            try:
-                resp = requests.get(image_url, stream=True)
-                resp.raise_for_status()
-                with open(local_path, 'wb') as imgf:
-                    for chunk in resp.iter_content(chunk_size=8192):
-                        imgf.write(chunk)
-                logger.info(f"Downloaded image: {local_path}")
-            except requests.RequestException as e:
-                logger.error(f"Failed to download image {image_url}: {e}")
-
-    def save_html_content(self, model_page_url: str, folder: str):
-        """モデルページのHTMLをフォルダ内に保存する."""
+def encrypt_and_upload_folder(self, local_folder: str) -> Optional[str]:
+    """
+    1. /home/user/app/encrypted 配下の古いファイルやフォルダを消去（オプション）
+    2. rclone mkdir cryptLocal:subfolder_label  (空ディレクトリを必ず作る)
+    3. rclone copy local_folder => cryptLocal:subfolder_label
+       → /home/user/app/encrypted/<暗号フォルダ>/ にコピー
+    4. そのフォルダを self.upload_folder() (＝HFにアップロード)
+    5. ローカル(平文フォルダ & 暗号フォルダ)削除
+    6. 最後に subfolder_label（論理上のフォルダ名）を return
+    """
+    if not os.path.exists(local_folder):
+        logger.error(f"encrypt_and_upload_folder: folder not found: {local_folder}")
+        return None
+
+    encrypted_base_dir = os.path.join(os.getcwd(), "encrypted")
+    os.makedirs(encrypted_base_dir, exist_ok=True)
+
+    # 1) 古い暗号ファイルやフォルダを削除
+    for item in os.listdir(encrypted_base_dir):
+        item_path = os.path.join(encrypted_base_dir, item)
         try:
-            resp = requests.get(model_page_url)
-            resp.raise_for_status()
-            html_path = os.path.join(folder, "page.html")
-            with open(html_path, 'w', encoding='utf-8') as f:
-                f.write(resp.text)
-            logger.info(f"Saved HTML: {html_path}")
+            if os.path.isfile(item_path) or os.path.islink(item_path):
+                os.remove(item_path)
+            else:
+                shutil.rmtree(item_path)
+            logger.info(f"[CLEANUP] Removed old encrypted item: {item_path}")
         except Exception as e:
-            logger.error(f"Error saving HTML content from {model_page_url}: {e}")
-
-    def save_model_info_json(self, model_info: dict, folder: str):
-        """モデル情報をJSONファイルとして保存."""
-        info_path = os.path.join(folder, "model_info.json")
-        try:
-            with open(info_path, 'w', encoding='utf-8') as f:
-                json.dump(model_info, f, indent=2)
-            logger.info(f"Saved model_info.json: {info_path}")
-        except Exception as e:
-            logger.error(f"Failed to save model info JSON: {e}")
-    # =========================================================================
-    # ここが重要:
-    #  - 最新バージョンはまとめて folder_name にダウンロード (一度に暗号化アップロード)
-    #  - 古いバージョンは1つずつダウンロード→暗号化→アップロード→削除 でストレージを節約
-    # =========================================================================
-    def download_and_process_versions(self, model_versions: list, folder: str):
-        """最新バージョンをまとめてダウンロード、old_versionsは1つずつアップして削除。"""
-
-        # 1) 最新バージョン (インデックス0) のファイルを folder にダウンロード
-        latest_version = model_versions[0]
-        logger.info(f"Processing latest version: {latest_version.get('name','(NoName)')}")
-
-        for file_info in latest_version.get("files", []):
-            download_url = file_info["downloadUrl"]
-            file_name = file_info["name"]
-            login_detected_count = 0
-
-            while login_detected_count < 5:
-                local_path = self.download_file(download_url, folder, file_name)
-                if local_path and "login" in os.listdir(folder):
-                    login_detected_count += 1
-                    os.remove(os.path.join(folder, "login"))
-                    logger.warning(f"'login' file found, retrying {file_name} ({login_detected_count}/5)")
-                else:
-                    break
-
-            if login_detected_count >= 5:
-                dummy_file_path = os.path.join(folder, f"{file_name}.download_failed")
-                try:
-                    with open(dummy_file_path, "w") as f:
-                        f.write("Download failed after 5 attempts.")
-                    logger.error(f"Failed to download {file_name}. Dummy file created: {dummy_file_path}")
-                except Exception as e:
-                    logger.error(f"Failed to create dummy file for {file_name}: {e}")
-
-        # 2) 古いバージョンがあれば 1つずつダウンロード→暗号化アップロード→削除
-        if len(model_versions) > 1:
-            old_versions_folder = os.path.join(folder, "old_versions")
-            os.makedirs(old_versions_folder, exist_ok=True)
-
-            for version in model_versions[1:]:
-                logger.info(f"Processing older version: {version.get('name','(NoName)')}")
-                for file_info in version.get("files", []):
-                    file_name = file_info["name"]
-                    download_url = file_info["downloadUrl"]
-                    login_detected_count = 0
-
-                    while login_detected_count < 5:
-                        local_path = self.download_file(download_url, old_versions_folder, file_name)
-                        if local_path and "login" in os.listdir(old_versions_folder):
-                            login_detected_count += 1
-                            os.remove(os.path.join(old_versions_folder, "login"))
-                            logger.warning(f"'login' file found, retrying {file_name} ({login_detected_count}/5)")
-                        else:
-                            break
-
-                    if login_detected_count >= 5:
-                        dummy_file_path = os.path.join(old_versions_folder, f"{file_name}.download_failed")
-                        try:
-                            with open(dummy_file_path, "w") as f:
-                                f.write("Download failed after 5 attempts.")
-                            logger.error(f"Failed to download {file_name}. Dummy file: {dummy_file_path}")
-                        except Exception as e:
-                            logger.error(f"Failed to create dummy file for {file_name}: {e}")
-                        continue  # 次のファイルへ
-
-                    # ===== ダウンロード成功した古いバージョンファイルを暗号化アップロード =====
-                    # フォルダごと暗号化でもいいですが、大容量を避けるためファイル単位で暗号化する例
-                    # ここでは "encrypt_and_upload_folder" の代わりに「encrypt_and_upload_file」するなど
-                    #   あるいはフォルダごとでもOK
-                    single_file_folder = os.path.join(old_versions_folder, "temp_single")
-                    os.makedirs(single_file_folder, exist_ok=True)
-                    try:
-                        # 移動して「このファイルだけ」が入ったフォルダを作る
-                        single_file_path = shutil.move(local_path, os.path.join(single_file_folder, file_name))
-                        # 暗号化アップロード
-                        self.encrypt_and_upload_folder(single_file_folder)
-                    except Exception as e:
-                        logger.error(f"Failed to encrypt/upload old version file: {e}")
-                    finally:
-                        # single_file_folder削除（encrypt_and_upload_folderで消えるはず）
-                        if os.path.exists(single_file_folder):
-                            shutil.rmtree(single_file_folder)
-                            logger.info(f"Removed temp_single folder {single_file_folder}")
-
-    def encrypt_and_upload_folder(self, local_folder: str) -> Optional[str]:
-        """
-        1. rclone mkdir cryptLocal:subfolder_label で空フォルダを作る
-        2. rclone copy local_folder => cryptLocal:subfolder_label --create-empty-src-dirs
-        3. そのフォルダを self.upload_folder() でアップ
-        4. ローカル平文フォルダ & 暗号フォルダを削除
-        5. subfolder_label を返す
-        """
-        if not os.path.exists(local_folder):
-            logger.error(f"encrypt_and_upload_folder: folder not found: {local_folder}")
-            return None
-
-        encrypted_base_dir = os.path.join(os.getcwd(), "encrypted")
-        os.makedirs(encrypted_base_dir, exist_ok=True)
-
-        # 既存の暗号ファイル削除（不要ならコメントアウト）
-        for item in os.listdir(encrypted_base_dir):
-            item_path = os.path.join(encrypted_base_dir, item)
-            try:
-                if os.path.isfile(item_path) or os.path.islink(item_path):
-                    os.remove(item_path)
-                else:
-                    shutil.rmtree(item_path)
-                logger.info(f"[CLEANUP] Removed old encrypted item: {item_path}")
-            except Exception as e:
-                logger.warning(f"[CLEANUP] Failed to remove {item_path}: {e}")
-
-        # サブフォルダ名を生成（enc_ + UUID）
-        subfolder_label = "enc_" + str(uuid.uuid4())[:8]
-
-        # ★ 追加: mkdir で先に空ディレクトリを作っておく
-        try:
-            subprocess.run(
-                ["rclone", "mkdir", f"cryptLocal:{subfolder_label}"],
-                check=True
-            )
-        except subprocess.CalledProcessError as e:
-            logger.error(f"rclone mkdir failed: {e}")
-            return None
-
-        # ★ --create-empty-src-dirs オプションを付けて copy
-        try:
-            subprocess.run(
-                [
-                    "rclone", 
-                    "copy", 
-                    local_folder, 
-                    f"cryptLocal:{subfolder_label}",
-                    "--create-empty-src-dirs"
-                ],
-                check=True
-            )
-        except subprocess.CalledProcessError as e:
-            logger.error(f"rclone copy failed: {e}")
-            return None
-
-        # 暗号フォルダのパス（必ずディレクトリができている前提）
-        enc_folder_path = os.path.join(encrypted_base_dir, subfolder_label)
-
-        if not os.path.isdir(enc_folder_path):
-            logger.error(f"[ERROR] {enc_folder_path} is not a directory, something is still off.")
-            return None
-
-        # アップロード
-        try:
-            self.upload_folder(enc_folder_path, path_in_repo=subfolder_label)
-            logger.info(f"Uploaded encrypted folder: {enc_folder_path}")
-        except Exception as e:
-            logger.error(f"Failed to upload encrypted folder {enc_folder_path}: {e}")
-
-        # ローカル削除
-        try:
-            shutil.rmtree(local_folder)
-            shutil.rmtree(enc_folder_path)
-            logger.info(f"Removed local folder: {local_folder} and {enc_folder_path}")
-        except Exception as e:
-            logger.error(f"Failed to remove local folders: {e}")
-
-        return subfolder_label
-
-    def upload_file(self, file_path: str, repo_id: Optional[str] = None, path_in_repo: Optional[str] = None):
-        if repo_id is None:
-            repo_id = self.repo_ids['current']
-        if path_in_repo is None:
-            path_in_repo = os.path.basename(file_path)
-        
-        max_retries = 5
-        attempt = 0
-        while attempt < max_retries:
-            try:
-                self.api.upload_file(
-                    path_or_fileobj=file_path,
-                    repo_id=repo_id,
-                    path_in_repo=path_in_repo
-                )
-                logger.info(f"Uploaded file: {file_path} to {repo_id} at {path_in_repo}")
-                return
-            except Exception as e:
-                attempt += 1
-                error_message = str(e)
-                if "over the limit of 100000 files" in error_message:
-                    logger.warning("File limit exceeded, creating a new repo.")
-                    self.repo_ids['current'] = self.increment_repo_name(self.repo_ids['current'])
-                    self.api.create_repo(repo_id=self.repo_ids['current'], private=True)
-                    attempt = 0
-                    continue
-                elif "you can retry this action in about 1 hour" in error_message:
-                    logger.warning("Rate limit hit. Waiting 1 hour...")
-                    time.sleep(3600)
-                    attempt -= 1
-                else:
-                    if attempt < max_retries:
-                        logger.warning(f"Failed to upload {file_path}, retry {attempt}/{max_retries}")
-                    else:
-                        logger.error(f"Failed after {max_retries} attempts: {e}")
-                        raise
-
-    def upload_file(self, file_path: str, repo_id: Optional[str] = None, path_in_repo: Optional[str] = None):
-        if repo_id is None:
-            repo_id = self.repo_ids['current']
-        if path_in_repo is None:
-            path_in_repo = os.path.basename(file_path)
-        
-        max_retries = 5
-        attempt = 0
-        while attempt < max_retries:
-            try:
-                self.api.upload_file(
-                    path_or_fileobj=file_path,
-                    repo_id=repo_id,
-                    path_in_repo=path_in_repo
-                )
-                logger.info(f"Uploaded file: {file_path} to {repo_id} at {path_in_repo}")
-                return
-            except Exception as e:
-                attempt += 1
-                error_message = str(e)
-                if "over the limit of 100000 files" in error_message:
-                    logger.warning("File limit exceeded, creating a new repo.")
-                    self.repo_ids['current'] = self.increment_repo_name(self.repo_ids['current'])
-                    self.api.create_repo(repo_id=self.repo_ids['current'], private=True)
-                    attempt = 0
-                    continue
-                elif "you can retry this action in about 1 hour" in error_message:
-                    logger.warning("Rate limit hit. Waiting 1 hour...")
-                    time.sleep(3600)
-                    attempt -= 1
-                else:
-                    if attempt < max_retries:
-                        logger.warning(f"Failed to upload {file_path}, retry {attempt}/{max_retries}")
-                    else:
-                        logger.error(f"Failed after {max_retries} attempts: {e}")
-                        raise
-
-    def upload_folder(self, folder_path: str, path_in_repo: Optional[str] = None):
-        """
-        フォルダを Hugging Face リポジトリに一括アップロード
-        """
-        if path_in_repo is None:
-            path_in_repo = os.path.basename(folder_path)
-
-        max_retries = 5
-        attempt = 0
-        while attempt < max_retries:
-            try:
-                self.api.upload_folder(
-                    folder_path=folder_path,
-                    repo_id=self.repo_ids['current'],
-                    path_in_repo=path_in_repo
-                )
-                logger.info(f"Uploaded folder: {folder_path} to {self.repo_ids['current']} at {path_in_repo}")
-                return
-            except Exception as e:
-                attempt += 1
-                error_message = str(e)
-                if "over the limit of 100000 files" in error_message:
-                    logger.warning("File limit exceeded, creating a new repo.")
-                    self.repo_ids['current'] = self.increment_repo_name(self.repo_ids['current'])
-                    self.api.create_repo(repo_id=self.repo_ids['current'], private=True)
-                    attempt = 0
-                    continue
-                elif "you can retry this action in about 1 hour" in error_message:
-                    logger.warning("Rate limit hit. Waiting 1 hour...")
-                    time.sleep(3600)
-                    attempt -= 1
-                else:
-                    if attempt < max_retries:
-                        logger.warning(f"Failed to upload folder {folder_path}, retry {attempt}/{max_retries}")
-                    else:
-                        logger.error(f"Failed after {max_retries} attempts: {e}")
-                        raise
-
-    @staticmethod
-    def increment_repo_name(repo_id: str) -> str:
-        """リポジトリ名の末尾の数字をインクリメントする。"""
-        match = re.search(r'(\d+)$', repo_id)
-        if match:
-            number = int(match.group(1)) + 1
-            new_repo_id = re.sub(r'\d+$', str(number), repo_id)
-        else:
-            new_repo_id = f"{repo_id}1"
-        return new_repo_id
-
-    def read_model_list(self) -> dict:
-        """モデルリストを読み込む。"""
-        model_list = {}
-        try:
-            with open(self.config.LIST_FILE, "r", encoding="utf-8") as f:
-                for line in f:
-                    line = line.strip()
-                    if line:
-                        parts = line.split(": ", 1)
-                        if len(parts) == 2:
-                            modelpage_name, model_hf_url = parts
-                            model_list[model_hf_url] = modelpage_name
-        except Exception as e:
-            logger.error(f"Failed to read model list: {e}")
-        return model_list
-
-    def get_repo_info(self, repo_id):
-        """リポジトリの情報を取得する。"""
-        try:
-            repo_info = self.api.repo_info(repo_id=repo_id, files_metadata=True)
-            file_paths = [sibling.rfilename for sibling in repo_info.siblings]
-            return file_paths
-        except Exception as e:
-            logger.error(f"Failed to get repo info for {repo_id}: {e}")
-            return []
-
-    def process_model(self, model_url: str):
-        """1つのモデルをダウンロードしてフォルダ丸ごと暗号化＆アップロードする."""
-        try:
-            # model_idを取得
-            model_id = model_url.rstrip("/").split("/")[-1]
-
-            # モデル情報を取得
-            model_info = self.get_model_info(model_id)
-            if not model_info or "modelVersions" not in model_info:
-                logger.error(f"No valid model info for ID {model_id}. Skipping.")
-                return
-
-            # バージョン一覧
-            versions = model_info["modelVersions"]
-            if not versions:
-                logger.warning(f"No modelVersions found for ID {model_id}.")
-                return
-
-            # ベースとなるフォルダ名
-            folder_name = model_info.get("name", "UnknownModel")
-            folder_name = re.sub(r'[\\/*?:"<>|]', '_', folder_name)
-            folder_name += "_" + str(uuid.uuid4())[:8]
-            os.makedirs(folder_name, exist_ok=True)
-
-            # (A) 最新バージョンファイル + (B) 画像 + (C) HTML + model_info.json
-            #    → 全部 folder_name に置く
-            #    → 後でまとめて encrypt_and_upload_folder
-            self.download_and_process_versions(versions, folder_name)
-            self.download_images(versions, folder_name)
-            self.save_html_content(f"{self.config.URLS['modelPage']}{model_id}", folder_name)
-            self.save_model_info_json(model_info, folder_name)
-
-            # ↑ この時点で「最新バージョン」の大ファイルが folder_name に残っている
-            #   しかし old_versions は既に1つずつ暗号化→削除済み
-
-            # まとめて暗号化 & アップロード
-            enc_subfolder = self.encrypt_and_upload_folder(folder_name)
-            if enc_subfolder is None:
-                enc_subfolder = "[ENCRYPT_FAILED]"
-
-            hf_enc_url = f"https://huggingface.co/{self.repo_ids['current']}/tree/main/{enc_subfolder}"
-            with open(self.config.LIST_FILE, "a", encoding="utf-8") as f:
-                f.write(f"{model_info.get('name', 'UnnamedModel')} (ID:{model_id}): {hf_enc_url}\n")
-
-        except Exception as e:
-            logger.error(f"Error in process_model ({model_url}): {e}")
-
-    async def crawl(self):
-        """モデルを定期的にチェックし、更新を行う。"""
-        while True:
-            try:
-                login(token=self.config.HUGGINGFACE_API_KEY, add_to_git_credential=True)
-
-                # model_list.logを最新化
-                model_list_path = hf_hub_download(
-                    repo_id=self.repo_ids['model_list'],
-                    filename=self.config.LIST_FILE
-                )
-                shutil.copyfile(model_list_path, f"./{self.config.LIST_FILE}")
-
-                # ログファイルを最新化
-                local_file_path = hf_hub_download(
-                    repo_id=self.repo_ids["log"],
-                    filename=self.config.LOG_FILE
-                )
-                shutil.copyfile(local_file_path, f"./{self.config.LOG_FILE}")
-
-                # ログを読み込み
-                with open(self.config.LOG_FILE, "r", encoding="utf-8") as file:
-                    lines = file.read().splitlines()
-                    old_models = json.loads(lines[0]) if len(lines) > 0 else []
-                    self.repo_ids["current"] = lines[1] if len(lines) > 1 else ""
-
-                # 新着モデル確認
-                r = requests.get(self.config.URLS["latest"], headers=self.config.HEADERS)
-                r.raise_for_status()
-                latest_models = r.json().get("items", [])
-                latest_model_ids = [m["id"] for m in latest_models if "id" in m]
-
-                new_models = list(set(latest_model_ids) - set(old_models))
-                if new_models:
-                    logger.info(f"New model IDs found: {new_models}")
-                    model_id = new_models[0]
-
-                    for attempt in range(1, 6):
-                        try:
-                            self.process_model(self.config.URLS["modelId"] + str(model_id))
-                            break
-                        except Exception as e:
-                            logger.error(f"Failed to process model {model_id} (attempt {attempt}/5): {e}")
-                            if attempt == 5:
-                                logger.error(f"Skipping model {model_id} after 5 failures.")
-                            else:
-                                await asyncio.sleep(2)
-
-                    # 新モデルをold_modelsに追加し、ログを更新
-                    old_models.append(model_id)
-                    with open(self.config.LOG_FILE, "w", encoding="utf-8") as f:
-                        f.write(json.dumps(old_models) + "\n")
-                        f.write(f"{self.repo_ids['current']}\n")
-                    logger.info(f"Updated log with new model ID: {model_id}")
-
-                    # ログファイル & model_list.logをアップロード
-                    self.upload_file(
-                        file_path=self.config.LOG_FILE,
-                        repo_id=self.repo_ids["log"],
-                        path_in_repo=self.config.LOG_FILE
-                    )
-                    self.upload_file(
-                        file_path=self.config.LIST_FILE,
-                        repo_id=self.repo_ids["model_list"],
-                        path_in_repo=self.config.LIST_FILE
-                    )
-                else:
-                    # 新着なし → ログを最新化してアップロードだけして待機
-                    with open(self.config.LOG_FILE, "w", encoding="utf-8") as f:
-                        f.write(json.dumps(latest_model_ids) + "\n")
-                        f.write(f"{self.repo_ids['current']}\n")
-                    logger.info(f"No new models. Updated log: {self.config.LOG_FILE}")
-                    self.upload_file(
-                        file_path=self.config.LOG_FILE,
-                        repo_id=self.repo_ids["log"],
-                        path_in_repo=self.config.LOG_FILE
-                    )
-                    logger.info("Uploaded log file.")
-                    await asyncio.sleep(60)
-                    continue
-
-            except Exception as e:
-                logger.error(f"Error in crawl loop: {e}")
-                await asyncio.sleep(300)
-
-
-# FastAPIアプリケーション
-config = Config()
-crawler = CivitAICrawler(config)
-app = crawler.app
+            logger.warning(f"[CLEANUP] Failed to remove {item_path}: {e}")
+
+    # 2) まず mkdir で「暗号リモート上に空ディレクトリ」を作る
+    subfolder_label = "enc_" + str(uuid.uuid4())[:8]
+    try:
+        subprocess.run(
+            ["rclone", "mkdir", f"cryptLocal:{subfolder_label}"],
+            check=True
+        )
+        logger.info(f"[OK] rclone mkdir cryptLocal:{subfolder_label}")
+    except subprocess.CalledProcessError as e:
+        logger.error(f"rclone mkdir failed: {e}")
+        return None
+
+    # 3) local_folder -> cryptLocal:subfolder_label にコピー
+    #    --create-empty-src-dirs: 空フォルダ構造も含めてコピー
+    try:
+        subprocess.run(
+            [
+                "rclone", "copy",
+                local_folder,
+                f"cryptLocal:{subfolder_label}",
+                "--create-empty-src-dirs"
+            ],
+            check=True
+        )
+        logger.info(f"[OK] rclone copy {local_folder} => cryptLocal:{subfolder_label}")
+    except subprocess.CalledProcessError as e:
+        logger.error(f"rclone copy failed: {e}")
+        return None
+
+    # cryptLocal:{subfolder_label} => 実際には /home/user/app/encrypted/<ランダム文字列> のフォルダになる
+    # それを差分検知で特定
+    before_dirs = set(os.listdir(encrypted_base_dir))
+    # mkdir/copy 直後に何らかのズレがあるかもしれないので再度 mkdir しなくてOK
+    # すでに上でやってるため、ここでやるなら:
+    # time.sleep(1) などで間を置いてみる手もある
+
+    after_dirs = set(os.listdir(encrypted_base_dir))
+    diff = after_dirs - before_dirs
+    #   もし diff が空なら既にあるフォルダに上書きコピーされた可能性
+    if not diff:
+        logger.error("[ERROR] No new directory appeared in ./encrypted after rclone copy.")
+        return None
+    if len(diff) > 1:
+        logger.warning(f"[WARN] Multiple new directories found: {diff}, picking the first one.")
+    enc_folder_name = diff.pop()
+    enc_folder_path = os.path.join(encrypted_base_dir, enc_folder_name)
+
+    if not os.path.isdir(enc_folder_path):
+        logger.error(f"[ERROR] {enc_folder_path} is not a directory.")
+        return None
+
+    # 4) アップロード (フォルダとして)
+    try:
+        self.upload_folder(enc_folder_path, path_in_repo=subfolder_label)
+        logger.info(f"Uploaded encrypted folder: {enc_folder_path}")
+    except Exception as e:
+        logger.error(f"Failed to upload encrypted folder {enc_folder_path}: {e}")
+
+    # 5) ローカル削除
+    try:
+        shutil.rmtree(local_folder)
+        shutil.rmtree(enc_folder_path)
+        logger.info(f"Removed local folder: {local_folder} and {enc_folder_path}")
+    except Exception as e:
+        logger.error(f"Failed to remove local folders: {e}")
+
+    # 6) 論理上のフォルダ名（rclone上の名称）を返す
+    return subfolder_label