Update main.py

main.py

@@ -14,14 +14,14 @@ import requests
 from bs4 import BeautifulSoup
 from fake_useragent import UserAgent
 from fastapi import FastAPI
-from huggingface_hub import HfApi, create_repo, hf_hub_download, login
+from huggingface_hub import HfApi, hf_hub_download, login
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
 class Config:
-    """設定用のクラス"""
+    """設定用クラス"""
     HUGGINGFACE_API_KEY = os.environ["HUGGINGFACE_API_KEY"]
     CIVITAI_API_TOKEN = os.environ["CIVITAI_API_TOKEN"]
     LOG_FILE = "civitai_backup.log"
@@ -35,24 +35,22 @@ class Config:
         "latest": "https://civitai.com/api/v1/models?sort=Newest",
         "modelPage": "https://civitai.com/models/",
         "modelId": "https://civitai.com/api/v1/models/",
-        "modelVersionId": "https://civitai.com/api/v1/model-versions/",
-        "hash": "https://civitai.com/api/v1/model-versions/by-hash/"
     }
     JST = datetime.timezone(datetime.timedelta(hours=9))
     UA = UserAgent()
     HEADERS = {
-        'Authorization': f'Bearer {CIVITAI_API_TOKEN}',
-        'User-Agent': 'civitai-crawler/1.0',
+        "Authorization": f'Bearer {CIVITAI_API_TOKEN}',
+        "User-Agent": "civitai-crawler/1.0",
         "Content-Type": "application/json"
     }
 
-    # rclone 用の追加設定
+    # rclone
     RCLONE_CONF_BASE64 = os.environ.get("RCLONE_CONF_BASE64", "")
     ENCRYPTED_DIR = "/home/user/app/encrypted"
 
 
 class CivitAICrawler:
-    """CivitAIからモデルをダウンロードし、Hugging Faceにアップロードするクラス"""
+    """CivitAIからダウンロード & Hugging Faceにアップロード"""
 
     def __init__(self, config: Config):
         self.config = config
@@ -61,165 +59,138 @@ class CivitAICrawler:
         self.repo_ids = self.config.REPO_IDS.copy()
         self.jst = self.config.JST
 
-        # rclone 設定の読み込み
         self.setup_rclone_conf()
         self.setup_routes()
 
     def setup_routes(self):
         @self.app.get("/")
-        def read_root():
-            now = str(datetime.datetime.now(self.jst))
-            description = (
-                f"CivitAIを定期的に周回し新規モデルを {self.repo_ids['current']} にバックアップするSpaceです。\n"
-                f"model_list.log や civitai_backup.log は暗号化しないでアップロードします。\n"
-                f"モデルのフォルダやファイルは暗号化してアップロードします。\n"
-                f"Status: {now} + currently running :D\n"
-            )
-            return description
+        def root():
+            now = datetime.datetime.now(self.jst)
+            return f"Status: {now} -- current repo: {self.repo_ids['current']}"
 
         @self.app.on_event("startup")
-        async def startup_event():
+        async def on_startup():
             asyncio.create_task(self.crawl())
 
-    # ============================================================================
-    # rclone 設定 & 暗号化アップロード処理
-    # ============================================================================
     def setup_rclone_conf(self):
         if not self.config.RCLONE_CONF_BASE64:
-            logger.warning("[WARN] RCLONE_CONF_BASE64 is empty. rclone may fail.")
+            logger.warning("No RCLONE_CONF_BASE64 found.")
             return
         os.makedirs(".rclone_config", exist_ok=True)
         conf_path = os.path.join(".rclone_config", "rclone.conf")
         with open(conf_path, "wb") as f:
             f.write(base64.b64decode(self.config.RCLONE_CONF_BASE64))
         os.environ["RCLONE_CONFIG"] = conf_path
-        logger.info(f"[INFO] rclone.conf created at: {conf_path}")
+        logger.info(f"rclone.conf created at: {conf_path}")
 
+    # ============================================================================
+    # rcloneで暗号化アップロード
+    # ============================================================================
     def encrypt_with_rclone(self, local_path: str):
-        """フォルダ or ファイルを cryptLocal: にコピーし、フォルダ名・ファイル名を暗号化"""
         if not os.path.exists(local_path):
-            raise FileNotFoundError(f"[ERROR] Local path not found: {local_path}")
-        # 事前に暗号先ディレクトリを掃除
+            raise FileNotFoundError(f"Local path not found: {local_path}")
         if os.path.isdir(self.config.ENCRYPTED_DIR):
             shutil.rmtree(self.config.ENCRYPTED_DIR, ignore_errors=True)
 
-        top_level_name = os.path.basename(local_path.rstrip("/")) or "unnamed"
-        cmd = ["rclone", "copy", local_path, f"cryptLocal:{top_level_name}", "-v"]
-        logger.info(f"[INFO] Running: {' '.join(cmd)}")
+        top_name = os.path.basename(local_path.rstrip("/")) or "unnamed"
+
+        cmd = [
+            "rclone", "copy",
+            local_path,
+            f"cryptLocal:{top_name}",
+            "-v"
+        ]
+        logger.info("Running: %s", " ".join(cmd))
         subprocess.run(cmd, check=True)
-        logger.info(f"[OK] rclone copy => cryptLocal:{top_level_name}")
 
         if not os.path.isdir(self.config.ENCRYPTED_DIR):
-            raise FileNotFoundError(
-                f"[ERROR] {self.config.ENCRYPTED_DIR} not found. Check your rclone config."
-            )
+            raise FileNotFoundError("Encrypted dir not found after rclone copy.")
 
-    def upload_encrypted_files(self, repo_id: str, base_path_in_repo: str = ""):
-        """self.config.ENCRYPTED_DIR 以下の暗号化済ファイルを再帰的にアップロード"""
+    def upload_encrypted_files(self, repo_id: str, path_in_repo: str = ""):
         max_retries = 5
         for root, dirs, files in os.walk(self.config.ENCRYPTED_DIR):
             for fn in files:
-                encrypted_file_path = os.path.join(root, fn)
-                if not os.path.isfile(encrypted_file_path):
-                    continue
-
-                relative_path = os.path.relpath(encrypted_file_path, self.config.ENCRYPTED_DIR)
-                upload_path_in_repo = os.path.join(base_path_in_repo, relative_path)
+                enc_file_path = os.path.join(root, fn)
+                rel_path = os.path.relpath(enc_file_path, self.config.ENCRYPTED_DIR)
+                upload_path = os.path.join(path_in_repo, rel_path)
 
                 attempt = 0
                 while attempt < max_retries:
                     try:
                         self.api.upload_file(
-                            path_or_fileobj=encrypted_file_path,
+                            path_or_fileobj=enc_file_path,
                             repo_id=repo_id,
-                            path_in_repo=upload_path_in_repo
+                            path_in_repo=upload_path
                         )
-                        logger.info(f"[OK] Uploaded => {repo_id}/{upload_path_in_repo}")
+                        logger.info("[OK] Uploaded => %s/%s", repo_id, upload_path)
                         break
                     except Exception as e:
                         attempt += 1
-                        error_message = str(e)
-                        # 429 Rate-limit with "in XX minutes"
-                        if "rate-limited" in error_message and "minutes" in error_message:
+                        msg = str(e)
+                        # rate-limit / file-limit handle
+                        if "rate-limited" in msg and "minutes" in msg:
                             import re
-                            match = re.search(r"in (\d+) minutes?", error_message)
-                            if match:
-                                minutes = int(match.group(1)) + 1
-                                logger.warning(f"Rate-limited. Waiting {minutes} minutes...")
-                                time.sleep(minutes * 60)
+                            m = re.search(r"in (\d+) minutes?", msg)
+                            if m:
+                                mins = int(m.group(1)) + 1
+                                logger.warning("Rate-limited. Wait %d minutes.", mins)
+                                time.sleep(mins * 60)
                                 attempt -= 1
                                 continue
-                        # 1時間待機
-                        if "you can retry this action in about 1 hour" in error_message:
-                            logger.warning("Encountered 'retry in 1 hour' error. Waiting 1 hour...")
+                        if "you can retry this action in about 1 hour" in msg:
+                            logger.warning("Encountered retry in 1 hour error, waiting..")
                             time.sleep(3600)
                             attempt -= 1
                             continue
-                        # ファイル上限
-                        if "over the limit of 100000 files" in error_message:
-                            logger.warning("Repository file limit exceeded. Creating a new repository...")
+                        if "over the limit of 100000 files" in msg:
+                            logger.warning("File limit exceeded. Creating new repo..")
                             self.repo_ids['current'] = self.increment_repo_name(self.repo_ids['current'])
                             self.api.create_repo(repo_id=self.repo_ids['current'], private=True)
                             attempt = 0
                             repo_id = self.repo_ids['current']
                             continue
-
                         if attempt < max_retries:
-                            logger.warning(
-                                f"Failed to upload {encrypted_file_path}, retry {attempt}/{max_retries}..."
-                            )
+                            logger.warning("Failed to upload, retry %d/%d..", attempt, max_retries)
                         else:
-                            logger.error(f"Failed to upload after {max_retries} attempts: {encrypted_file_path}")
+                            logger.error("Failed after %d attempts: %s", max_retries, enc_file_path)
                             raise
 
     def upload_folder_encrypted(self, folder_path: str, repo_id: Optional[str] = None, path_in_repo: str = ""):
-        """フォルダを丸ごと暗号化してアップロード (=フォルダ名も暗号化)"""
         if not repo_id:
             repo_id = self.repo_ids['current']
-
         self.encrypt_with_rclone(folder_path)
-        self.upload_encrypted_files(repo_id, base_path_in_repo=path_in_repo)
-
-        # 暗号化フォルダを削除
+        self.upload_encrypted_files(repo_id, path_in_repo=path_in_repo)
         if os.path.isdir(self.config.ENCRYPTED_DIR):
             shutil.rmtree(self.config.ENCRYPTED_DIR, ignore_errors=True)
 
-    # ============================================================================
-    # 単一ファイルを暗号化アップロードしてローカル削除 (old_versions用)
-    # ============================================================================
     def upload_file_encrypted_one_by_one(self, file_path: str, repo_id: Optional[str] = None, path_in_repo: str = ""):
-        """
-        単一ファイルを暗号化アップロードし、アップロード後にローカルファイルを削除。
-        """
+        """古いバージョン用: ファイルを暗号化アップロード → ローカル削除。"""
         if not repo_id:
             repo_id = self.repo_ids['current']
 
         self.encrypt_with_rclone(file_path)
-        self.upload_encrypted_files(repo_id, base_path_in_repo=path_in_repo)
-
-        # 暗号化ディレクトリを削除
+        self.upload_encrypted_files(repo_id, path_in_repo)
         if os.path.isdir(self.config.ENCRYPTED_DIR):
             shutil.rmtree(self.config.ENCRYPTED_DIR, ignore_errors=True)
-        # ローカルの実ファイル削除
         if os.path.exists(file_path):
             os.remove(file_path)
 
     @staticmethod
     def increment_repo_name(repo_id: str) -> str:
-        match = re.search(r'(\d+)$', repo_id)
-        if match:
-            number = int(match.group(1)) + 1
-            return re.sub(r'\d+$', str(number), repo_id)
+        m = re.search(r'(\d+)$', repo_id)
+        if m:
+            num = int(m.group(1)) + 1
+            return re.sub(r'\d+$', str(num), repo_id)
         else:
-            return f"{repo_id}1"
+            return repo_id + "1"
 
     # ============================================================================
-    # ログや model_list.log は生アップロード
+    # 生ファイルアップロード(ログやmodel_listなど)
     # ============================================================================
     def upload_file_raw(self, file_path: str, repo_id: Optional[str] = None, path_in_repo: Optional[str] = None):
-        if repo_id is None:
+        if not repo_id:
             repo_id = self.repo_ids['current']
-        if path_in_repo is None:
+        if not path_in_repo:
             path_in_repo = os.path.basename(file_path)
 
         max_retries = 5
@@ -231,320 +202,287 @@ class CivitAICrawler:
                     repo_id=repo_id,
                     path_in_repo=path_in_repo
                 )
-                logger.info(f"[OK] Uploaded {file_path} => {repo_id}/{path_in_repo}")
+                logger.info("[OK] Uploaded raw %s => %s/%s", file_path, repo_id, path_in_repo)
                 return
             except Exception as e:
                 attempt += 1
-                error_message = str(e)
-                if "over the limit of 100000 files" in error_message:
-                    logger.warning("Repository file limit exceeded, creating a new repository.")
+                msg = str(e)
+                if "over the limit of 100000 files" in msg:
+                    logger.warning("File limit exceeded, creating new repo..")
                     self.repo_ids['current'] = self.increment_repo_name(self.repo_ids['current'])
                     self.api.create_repo(repo_id=self.repo_ids['current'], private=True)
                     attempt = 0
                     repo_id = self.repo_ids['current']
                     continue
-                elif "you can retry this action in about 1 hour" in error_message:
-                    logger.warning("Encountered 'retry in 1 hour' error. Waiting 1 hour before retrying...")
+                if "you can retry this action in about 1 hour" in msg:
+                    logger.warning("Rate-limited 1h, waiting..")
                     time.sleep(3600)
                     attempt -= 1
                 else:
                     if attempt < max_retries:
-                        logger.warning(f"Failed to upload raw file {file_path}, retry {attempt}/{max_retries}...")
+                        logger.warning("Failed raw upload attempt %d/%d..", attempt, max_retries)
                     else:
-                        logger.error(f"Failed to upload raw file after {max_retries} attempts: {file_path}")
+                        logger.error("Failed raw upload after %d attempts: %s", max_retries, file_path)
                         raise
 
     # ============================================================================
-    # ダウンロード処理
+    # ダウンロード
     # ============================================================================
     @staticmethod
-    def get_filename_from_cd(content_disposition: Optional[str], default_name: str) -> str:
-        if content_disposition:
-            parts = content_disposition.split(';')
-            for part in parts:
-                if "filename=" in part:
-                    return part.split("=")[1].strip().strip('"')
+    def get_filename_from_cd(cd: Optional[str], default_name: str) -> str:
+        if cd:
+            parts = cd.split(";")
+            for p in parts:
+                if "filename=" in p:
+                    return p.split("=")[1].strip().strip('"')
         return default_name
 
-    def download_file(self, url: str, destination_folder: str, default_name: str):
+    def download_file(self, url: str, dest_folder: str, default_name: str):
         try:
-            response = requests.get(url, headers=self.config.HEADERS, stream=True)
-            response.raise_for_status()
+            resp = requests.get(url, headers=self.config.HEADERS, stream=True)
+            resp.raise_for_status()
         except requests.RequestException as e:
-            logger.error(f"Failed to download file from {url}: {e}")
+            logger.error("Failed to DL from %s: %s", url, e)
             return None
 
-        filename = self.get_filename_from_cd(response.headers.get('content-disposition'), default_name)
-        file_path = os.path.join(destination_folder, filename)
+        filename = self.get_filename_from_cd(resp.headers.get('content-disposition'), default_name)
+        local_path = os.path.join(dest_folder, filename)
+        os.makedirs(dest_folder, exist_ok=True)
 
-        with open(file_path, 'wb') as file:
-            for chunk in response.iter_content(chunk_size=8192):
-                file.write(chunk)
+        with open(local_path, "wb") as f:
+            for chunk in resp.iter_content(chunk_size=8192):
+                f.write(chunk)
 
-        logger.info(f"Download completed: {file_path}")
-        return file_path
+        logger.info("Downloaded: %s", local_path)
+        return local_path
 
     # ============================================================================
-    # 古いバージョンのみ1ファイルずつアップロード
+    # 旧バージョンファイルだけ1ファイルずつアップロード→削除
     # ============================================================================
-    def download_old_versions_one_by_one(self, version_list: list, folder: str):
-        """version_list[1:] を対象に、モデルファイルを 1ファイルDL→upload→削除 を繰り返す"""
-        if len(version_list) <= 1:
+    def download_old_versions_one_by_one(self, model_versions: list, folder: str):
+        """model_versions[1:] を対象にファイルDL→暗号化UL→削除."""
+        if len(model_versions) <= 1:
             return
-
-        old_versions_folder = os.path.join(folder, "old_versions")
-        os.makedirs(old_versions_folder, exist_ok=True)
-
-        for version in version_list[1:]:
-            for file_info in version.get("files", []):
-                download_url = file_info["downloadUrl"]
-                file_name = file_info["name"]
-
-                local_path = self.download_file(download_url, old_versions_folder, file_name)
-                if not local_path or not os.path.exists(local_path):
-                    logger.error(f"Failed to download or file not found: {file_name}")
-                    continue
-
-                # 1つアップロードして削除
-                # path_in_repo を空文字にすればフォルダ名も暗号化される（トップレベル）
-                # もしサブフォルダにまとめたいなら "old_versions" とか指定する
-                self.upload_file_encrypted_one_by_one(local_path, path_in_repo="")
-
-        # old_versions フォルダ内は空になったはずなので削除
-        if os.path.exists(old_versions_folder):
-            shutil.rmtree(old_versions_folder, ignore_errors=True)
+        old_dir = os.path.join(folder, "old_versions")
+        os.makedirs(old_dir, exist_ok=True)
+
+        for ver in model_versions[1:]:
+            for f_info in ver.get("files", []):
+                url = f_info["downloadUrl"]
+                fname = f_info["name"]
+                local_path = self.download_file(url, old_dir, fname)
+                if local_path and os.path.exists(local_path):
+                    self.upload_file_encrypted_one_by_one(local_path)
+        if os.path.exists(old_dir):
+            shutil.rmtree(old_dir, ignore_errors=True)
 
     # ============================================================================
-    # 従来どおり「最新バージョンのファイル一式 + images」フォルダを一括DL→アップロード
+    # 通常の(最新)モデルファイル & images フォルダを一括DL→アップロード
     # ============================================================================
-    def download_model(self, model_versions: list, folder: str):
-        """最新バージョンを一括ダウンロード (フォルダにまとめる)"""
-        latest_version = model_versions[0]
-        latest_files = latest_version["files"]
-
-        for file_info in latest_files:
-            download_url = file_info["downloadUrl"]
-            file_name = file_info["name"]
-            local_path = self.download_file(download_url, folder, file_name)
-            if local_path and os.path.exists(local_path):
-                logger.info(f"Downloaded {file_name}")
-            else:
-                logger.warning(f"Could not download {file_name}")
+    def download_latest_files(self, model_versions: list, folder: str):
+        latest_files = model_versions[0].get("files", [])
+        os.makedirs(folder, exist_ok=True)
+        # 最新ファイル
+        for f_info in latest_files:
+            url = f_info["downloadUrl"]
+            fname = f_info["name"]
+            self.download_file(url, folder, fname)
 
     def download_images(self, model_versions: list, folder: str):
         images_folder = os.path.join(folder, "images")
         os.makedirs(images_folder, exist_ok=True)
 
-        images = []
-        for version in model_versions:
-            for img in version.get("images", []):
-                images.append(img["url"])
-
-        for image_url in images:
-            image_name = os.path.basename(image_url) + ".png"
-            local_path = os.path.join(images_folder, image_name)
-            try:
-                resp = requests.get(image_url, stream=True)
-                resp.raise_for_status()
-                with open(local_path, "wb") as f:
-                    for chunk in resp.iter_content(chunk_size=8192):
-                        f.write(chunk)
-                logger.info(f"Downloaded image: {local_path}")
-            except Exception as e:
-                logger.error(f"Error downloading image {image_url}: {e}")
+        for ver in model_versions:
+            for img in ver.get("images", []):
+                img_url = img["url"]
+                # 例: 49945407.jpeg --> "49945407.jpeg.png"
+                image_name = os.path.basename(img_url) + ".png"
+                self.download_file(img_url, images_folder, image_name)
 
     def save_html_content(self, url: str, folder: str):
         try:
-            response = requests.get(url)
-            response.raise_for_status()
-            html_path = os.path.join(folder, os.path.basename(folder) + ".html")
-            with open(html_path, 'w', encoding='utf-8') as file:
-                file.write(response.text)
+            resp = requests.get(url)
+            resp.raise_for_status()
+            html_name = os.path.basename(folder) + ".html"
+            html_path = os.path.join(folder, html_name)
+            with open(html_path, "w", encoding="utf-8") as f:
+                f.write(resp.text)
+            logger.info("Saved HTML => %s", html_path)
         except Exception as e:
-            logger.error(f"Error saving HTML content for URL {url}: {e}")
+            logger.error("Failed to save HTML from %s: %s", url, e)
 
-    @staticmethod
-    def save_model_info(model_info: dict, folder: str):
-        with open(os.path.join(folder, "model_info.json"), "w", encoding="utf-8") as file:
-            json.dump(model_info, file, indent=2)
+    def save_model_info(self, model_info: dict, folder: str):
+        info_path = os.path.join(folder, "model_info.json")
+        try:
+            with open(info_path, "w", encoding="utf-8") as f:
+                json.dump(model_info, f, indent=2)
+            logger.info("Saved model_info => %s", info_path)
+        except Exception as e:
+            logger.error("Failed to save model_info: %s", e)
 
     # ============================================================================
-    # model_list.log
+    # model_list.log の読み書き
     # ============================================================================
     def read_model_list(self):
-        model_list = {}
+        ret = {}
+        if not os.path.exists(self.config.LIST_FILE):
+            return ret
         try:
             with open(self.config.LIST_FILE, "r", encoding="utf-8") as f:
                 for line in f:
                     line = line.strip()
                     if not line:
                         continue
+                    # 例: "111123: https://huggingface.co/xxx"
                     parts = line.split(": ", 1)
                     if len(parts) == 2:
-                        stored_id, stored_url = parts
-                        model_list[stored_id] = stored_url
+                        key, val = parts
+                        ret[key] = val
         except Exception as e:
-            logger.error(f"Failed to read model list: {e}")
-        return model_list
+            logger.error("Failed to read model_list.log: %s", e)
+        return ret
+
+    def append_model_list(self, key: str, val: str):
+        """'key: val' を model_list.log の末尾に追加."""
+        with open(self.config.LIST_FILE, "a", encoding="utf-8") as f:
+            f.write(f"{key}: {val}\n")
+        logger.info("Appended to model_list.log => '%s: %s'", key, val)
 
     # ============================================================================
-    # model 情報取得
+    # get_model_info
     # ============================================================================
-    def get_model_info(self, model_id: str) -> dict:
+    def get_model_info(self, model_id: str):
+        url = self.config.URLS["modelId"] + str(model_id)
         try:
-            url = self.config.URLS["modelId"] + str(model_id)
             resp = requests.get(url, headers=self.config.HEADERS)
             resp.raise_for_status()
             return resp.json()
-        except requests.RequestException as e:
-            logger.error(f"Failed to retrieve model info for ID {model_id}: {e}")
+        except Exception as e:
+            logger.error("Failed to get model info for %s: %s", model_id, e)
             return {}
 
     # ============================================================================
-    # メイン処理: 最新ファイル + images はフォルダごとアップロード。old_versions は1ファイルずつ。
+    # メイン: 1モデルを処理
     # ============================================================================
     def process_model(self, model_url: str):
         try:
             model_id = model_url.rstrip("/").split("/")[-1]
             model_info = self.get_model_info(model_id)
             if not model_info:
-                logger.error(f"No model_info returned for {model_id}")
+                logger.error("No model_info for %s", model_id)
+                return
+            vers = model_info.get("modelVersions", [])
+            if not vers:
+                logger.error("No modelVersions for %s", model_id)
                 return
 
-            model_versions = model_info.get("modelVersions", [])
-            if not model_versions:
-                logger.error(f"No modelVersions in model info {model_id}")
+            # 既にアップしたかどうか model_list.log で判定(例: model_id で判定)
+            model_list = self.read_model_list()
+            if model_id in model_list:
+                logger.info("Model ID %s is already in model_list. Skipping..", model_id)
                 return
 
-            latest_version = model_versions[0]
-            model_file = next((file for file in latest_version["files"] if file.get('type') == 'Model'), None)
-            if model_file:
-                latest_filename = model_file['name']
-                folder = os.path.splitext(latest_filename)[0]
+            # まずフォルダ名
+            latest_files = vers[0].get("files", [])
+            if latest_files:
+                main_file = next((f for f in latest_files if f.get("type") == "Model"), latest_files[0])
+                main_name = main_file["name"]
+                folder = os.path.splitext(main_name)[0]
             else:
-                first_file = latest_version["files"][0]
-                latest_filename = first_file['name']
-                folder = os.path.splitext(latest_filename)[0]
-                logger.warning(f"No 'Model' type file found for model ID {model_id}. Using first file's name.")
+                folder = f"model_{model_id}"
 
+            # 1) 最新バージョン & images をまとめて download
             os.makedirs(folder, exist_ok=True)
-
-            # すでにアップ済みかどうか model_list.log でチェック (モデル名ベース)
-            model_list = self.read_model_list()
-            modelpage_name = model_info.get("name", f"Model_{model_id}")
-            if modelpage_name in model_list.values():
-                logger.info(f"Model '{modelpage_name}' already in model_list. Skipping.")
-                # 必要ならreturn
-
-            # 最新バージョン (まとめてダウンロード)
-            self.download_model(model_versions, folder)
-
-            # 画像 (imagesフォルダまるごとダウンロード)
-            self.download_images(model_versions, folder)
+            self.download_latest_files(vers, folder)
+            self.download_images(vers, folder)
 
             # HTML & model_info.json
             self.save_html_content(self.config.URLS["modelPage"] + str(model_id), folder)
             self.save_model_info(model_info, folder)
 
-            # 古いバージョンのみ「1つずつアップロード＆削除」
-            self.download_old_versions_one_by_one(model_versions, folder)
-
-            # ↑で old_versions は空になった → あとはフォルダに残っているのは
-            #   最新バージョンファイル・imagesフォルダ・model_info.json・HTML など
-
-            # "folder" 自体を暗号化アップロード (= images フォルダごとアップロード)
-            #   path_in_repo を "" にすればフォルダ名も暗号化される
-            self.upload_folder_encrypted(folder, path_in_repo="")
+            # 2) old_versions を1ファイルずつアップロード
+            self.download_old_versions_one_by_one(vers, folder)
 
-            # ローカルフォルダ削除
-            if os.path.exists(folder):
-                shutil.rmtree(folder)
+            # 3) 上記が終わると old_versions は消えている。
+            #    あとはフォルダに最新バージョンのファイル＆images etc.が残っているので
+            #    フォルダ一括で暗号化アップロード
+            encrypted_top_name = self.upload_folder_encrypted(folder)
+            shutil.rmtree(folder, ignore_errors=True)
 
-            # model_list.log に追記
-            # HF上では folder名 も暗号化されるが、ここでは元の "modelpage_name" と
-            # HFへのトップフォルダ参照URLを書く
-            model_hf_url = f"https://huggingface.co/{self.repo_ids['current']}/tree/main"
-            with open(self.config.LIST_FILE, "a", encoding="utf-8") as f:
-                f.write(f"{modelpage_name}: {model_hf_url}\n")
+            # model_list.log に書き込み (例: "111123: https://huggingface.co/xxx/tree/main")
+            #   → 実際にフォルダ暗号化でトップレベル名が変わってるはずなので、
+            #      ざっくり "/tree/main" を書く
+            hf_url = f"https://huggingface.co/{self.repo_ids['current']}/tree/main/{encrypted_top_name}"
+            self.append_model_list(model_id, hf_url)
 
         except Exception as e:
-            logger.error(f"Unexpected error processing model ({model_url}): {e}")
+            logger.error("Unexpected error in process_model(%s): %s", model_url, e)
 
     # ============================================================================
-    # crawl
+    # メインループ
     # ============================================================================
     async def crawl(self):
         while True:
             try:
                 login(token=self.config.HUGGINGFACE_API_KEY, add_to_git_credential=True)
+                # ダウンロードしてローカルにコピー
+                ml_path = hf_hub_download(repo_id=self.repo_ids["model_list"], filename=self.config.LIST_FILE)
+                shutil.copyfile(ml_path, self.config.LIST_FILE)
 
-                # 最新の model_list.log & civitai_backup.log をダウンロード
-                model_list_path = hf_hub_download(repo_id=self.repo_ids['model_list'], filename=self.config.LIST_FILE)
-                shutil.copyfile(model_list_path, f"./{self.config.LIST_FILE}")
+                log_path = hf_hub_download(repo_id=self.repo_ids["log"], filename=self.config.LOG_FILE)
+                shutil.copyfile(log_path, self.config.LOG_FILE)
 
-                local_file_path = hf_hub_download(repo_id=self.repo_ids["log"], filename=self.config.LOG_FILE)
-                shutil.copyfile(local_file_path, f"./{self.config.LOG_FILE}")
-
-                # ログ読み込み
-                with open(self.config.LOG_FILE, "r", encoding="utf-8") as file:
-                    lines = file.read().splitlines()
+                with open(self.config.LOG_FILE, "r", encoding="utf-8") as f:
+                    lines = f.read().splitlines()
                     old_models = json.loads(lines[0]) if len(lines) > 0 else []
                     self.repo_ids["current"] = lines[1] if len(lines) > 1 else ""
 
-                # 新着モデル確認
-                response = requests.get(self.config.URLS["latest"], headers=self.config.HEADERS)
-                response.raise_for_status()
-                latest_models = response.json().get("items", [])
-                latest_model_ids = [m["id"] for m in latest_models if "id" in m]
-
-                # 差集合
-                new_models = list(set(latest_model_ids) - set(old_models))
-
-                if new_models:
-                    logger.info(f"New models found: {new_models}")
-                    model_id = new_models[0]
+                # 最新モデルチェック
+                r = requests.get(self.config.URLS["latest"], headers=self.config.HEADERS)
+                r.raise_for_status()
+                items = r.json().get("items", [])
+                latest_ids = [it["id"] for it in items if "id" in it]
 
+                new_ids = list(set(latest_ids) - set(old_models))
+                if new_ids:
+                    logger.info("New models found: %s", new_ids)
+                    target_id = new_ids[0]
                     for attempt in range(1, 6):
                         try:
-                            self.process_model(f"{self.config.URLS['modelId']}{model_id}")
+                            self.process_model(f"{self.config.URLS['modelId']}{target_id}")
                             break
                         except Exception as e:
-                            logger.error(f"Failed to process model ID {model_id} (Attempt {attempt}/5): {e}")
+                            logger.error("Failed to process ID %s (attempt %d/5): %s", target_id, attempt, e)
                             if attempt == 5:
-                                logger.error(f"Skipping model ID {model_id} after 5 failed attempts.")
+                                logger.error("Skipping ID %s after 5 attempts", target_id)
                             else:
                                 await asyncio.sleep(2)
                 else:
                     # 新モデルなし
                     with open(self.config.LOG_FILE, "w", encoding="utf-8") as f:
-                        f.write(json.dumps(latest_model_ids) + "\n")
-                        f.write(f"{self.repo_ids['current']}\n")
-                    logger.info(f"Updated log file: {self.config.LOG_FILE}")
-
+                        f.write(json.dumps(latest_ids) + "\n")
+                        f.write(self.repo_ids["current"] + "\n")
                     self.upload_file_raw(self.config.LOG_FILE, self.repo_ids["log"], self.config.LOG_FILE)
-                    logger.info("Uploaded log file to repository (unencrypted).")
-
-                    logger.info("No new models found.")
+                    logger.info("No new models found. Sleep 60s..")
                     await asyncio.sleep(60)
                     continue
 
-                # 1件アップロードに成功したら old_models に追加
-                old_models.append(model_id)
+                # 成功モデルIDを old_models に追加 & civitai_backup.log 書き込み
+                old_models.append(target_id)
                 with open(self.config.LOG_FILE, "w", encoding="utf-8") as f:
                     f.write(json.dumps(old_models) + "\n")
-                    f.write(f"{self.repo_ids['current']}\n")
-                logger.info(f"Updated log file with new model ID: {model_id}")
+                    f.write(self.repo_ids["current"] + "\n")
+                logger.info("Updated log with new ID: %s", target_id)
 
-                # ログと model_list.log をアップ
+                # logs をアップ
                 self.upload_file_raw(self.config.LOG_FILE, self.repo_ids["log"], self.config.LOG_FILE)
                 self.upload_file_raw(self.config.LIST_FILE, self.repo_ids["model_list"], self.config.LIST_FILE)
 
             except Exception as e:
-                logger.error(f"Error during crawling: {e}")
+                logger.error("Error in crawl: %s", e)
                 await asyncio.sleep(300)
 
 
-# 実行
 config = Config()
 crawler = CivitAICrawler(config)
 app = crawler.app