Update main.py

main.py

@@ -1,97 +1,645 @@
-def encrypt_and_upload_folder(self, local_folder: str) -> Optional[str]:
-    """
-    1. /home/user/app/encrypted 配下の古いファイルやフォルダを消去（オプション）
-    2. rclone mkdir cryptLocal:subfolder_label  (空ディレクトリを必ず作る)
-    3. rclone copy local_folder => cryptLocal:subfolder_label
-       → /home/user/app/encrypted/<暗号フォルダ>/ にコピー
-    4. そのフォルダを self.upload_folder() (＝HFにアップロード)
-    5. ローカル(平文フォルダ & 暗号フォルダ)削除
-    6. 最後に subfolder_label（論理上のフォルダ名）を return
-    """
-    if not os.path.exists(local_folder):
-        logger.error(f"encrypt_and_upload_folder: folder not found: {local_folder}")
-        return None
-
-    encrypted_base_dir = os.path.join(os.getcwd(), "encrypted")
-    os.makedirs(encrypted_base_dir, exist_ok=True)
-
-    # 1) 古い暗号ファイルやフォルダを削除
-    for item in os.listdir(encrypted_base_dir):
-        item_path = os.path.join(encrypted_base_dir, item)
+import asyncio
+import datetime
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+import time
+import uuid
+from typing import Optional
+
+import requests
+from bs4 import BeautifulSoup
+from fake_useragent import UserAgent
+from fastapi import FastAPI
+from huggingface_hub import HfApi, hf_hub_download, login
+
+# ロギングの設定
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class Config:
+    """設定用のクラス"""
+    HUGGINGFACE_API_KEY = os.environ["HUGGINGFACE_API_KEY"]
+    CIVITAI_API_TOKEN = os.environ["CIVITAI_API_TOKEN"]
+    LOG_FILE = "civitai_backup.log"
+    LIST_FILE = "model_list.log"
+    REPO_IDS = {
+        "log": "ttttdiva/CivitAI_log_test",
+        "model_list": "ttttdiva/CivitAI_model_info_test",
+        "current": ""
+    }
+    URLS = {
+        "latest": "https://civitai.com/api/v1/models?sort=Newest",
+        "modelPage": "https://civitai.com/models/",
+        "modelId": "https://civitai.com/api/v1/models/",
+        "modelVersionId": "https://civitai.com/api/v1/model-versions/",
+        "hash": "https://civitai.com/api/v1/model-versions/by-hash/"
+    }
+    JST = datetime.timezone(datetime.timedelta(hours=9))
+    UA = UserAgent()
+    HEADERS = {
+        'Authorization': f'Bearer {CIVITAI_API_TOKEN}',
+        'User-Agent': UA.random,
+        "Content-Type": "application/json"
+    }
+
+
+class CivitAICrawler:
+    """CivitAIからモデルをダウンロードし、Hugging Faceにアップロードするクラス"""
+
+    def __init__(self, config: Config):
+        import base64
+
+        rclone_conf_base64 = os.environ.get("RCLONE_CONF_BASE64")
+        if rclone_conf_base64:
+            # カレントディレクトリ配下に .rclone_config ディレクトリを作成
+            config_dir = os.path.join(os.getcwd(), ".rclone_config")
+            os.makedirs(config_dir, exist_ok=True)
+
+            conf_path = os.path.join(config_dir, "rclone.conf")
+            with open(conf_path, "wb") as f:
+                f.write(base64.b64decode(rclone_conf_base64))
+
+            # rclone がここを参照するように設定
+            os.environ["RCLONE_CONFIG"] = conf_path
+            logger.info(f"[INFO] Created rclone.conf at {conf_path}")
+        else:
+            logger.warning("[WARN] RCLONE_CONF_BASE64 not found; rclone may fail.")
+
+        self.config = config
+        self.api = HfApi()
+        self.app = FastAPI()
+        self.repo_ids = self.config.REPO_IDS.copy()
+        self.jst = self.config.JST
+        self.setup_routes()
+
+    def setup_routes(self):
+        """FastAPIのルーティングを設定する。"""
+        @self.app.get("/")
+        def read_root():
+            now = str(datetime.datetime.now(self.jst))
+            description = f"""
+            CivitAIを定期的に周回し新規モデルを {self.repo_ids['current']} にバックアップするspaceです。
+            モデル一覧は https://huggingface.co/{self.repo_ids['model_list']}/blob/main/model_list.log を参照してください。
+            Status: {now} + currently running :D
+            """
+            return description
+
+        @self.app.on_event("startup")
+        async def startup_event():
+            asyncio.create_task(self.crawl())
+
+    @staticmethod
+    def get_filename_from_cd(content_disposition: Optional[str], default_name: str) -> str:
+        """Content-Dispositionヘッダーからファイル名を取得する。"""
+        if content_disposition:
+            parts = content_disposition.split(';')
+            for part in parts:
+                if "filename=" in part:
+                    return part.split("=")[1].strip().strip('"')
+        return default_name
+
+    def download_file(self, url: str, destination_folder: str, default_name: str) -> Optional[str]:
+        """指定されたURLからファイルをダウンロードし、指定されたフォルダに保存する。"""
+        try:
+            response = requests.get(url, headers=self.config.HEADERS, stream=True)
+            response.raise_for_status()
+        except requests.RequestException as e:
+            logger.error(f"Failed to download file from {url}: {e}")
+            return None
+
+        filename = self.get_filename_from_cd(response.headers.get('content-disposition'), default_name)
+        file_path = os.path.join(destination_folder, filename)
+
+        # ダウンロードとファイル保存処理
+        with open(file_path, 'wb') as file:
+            for chunk in response.iter_content(chunk_size=8192):
+                file.write(chunk)
+        logger.info(f"Downloaded: {file_path}")
+        return file_path
+
+    def get_model_info(self, model_id: str) -> dict:
+        """モデルの情報を取得する。"""
+        try:
+            response = requests.get(self.config.URLS["modelId"] + str(model_id), headers=self.config.HEADERS)
+            response.raise_for_status()
+            return response.json()
+        except requests.RequestException as e:
+            logger.error(f"Failed to retrieve model info for ID {model_id}: {e}")
+            return {}
+
+    def download_images(self, model_versions: list, folder: str):
+        """画像を images フォルダにまとめてダウンロードする."""
+        images_folder = os.path.join(folder, "images")
+        os.makedirs(images_folder, exist_ok=True)
+        images = []
+        for version in model_versions:
+            for img in version.get("images", []):
+                images.append(img["url"])
+        for image_url in images:
+            image_name = os.path.basename(image_url)
+            local_path = os.path.join(images_folder, image_name)
+            try:
+                resp = requests.get(image_url, stream=True)
+                resp.raise_for_status()
+                with open(local_path, 'wb') as imgf:
+                    for chunk in resp.iter_content(chunk_size=8192):
+                        imgf.write(chunk)
+                logger.info(f"Downloaded image: {local_path}")
+            except requests.RequestException as e:
+                logger.error(f"Failed to download image {image_url}: {e}")
+
+    def save_html_content(self, model_page_url: str, folder: str):
+        """モデルページのHTMLをフォルダ内に保存する."""
         try:
-            if os.path.isfile(item_path) or os.path.islink(item_path):
-                os.remove(item_path)
-            else:
-                shutil.rmtree(item_path)
-            logger.info(f"[CLEANUP] Removed old encrypted item: {item_path}")
+            resp = requests.get(model_page_url)
+            resp.raise_for_status()
+            html_path = os.path.join(folder, "page.html")
+            with open(html_path, 'w', encoding='utf-8') as f:
+                f.write(resp.text)
+            logger.info(f"Saved HTML: {html_path}")
         except Exception as e:
-            logger.warning(f"[CLEANUP] Failed to remove {item_path}: {e}")
-
-    # 2) まず mkdir で「暗号リモート上に空ディレクトリ」を作る
-    subfolder_label = "enc_" + str(uuid.uuid4())[:8]
-    try:
-        subprocess.run(
-            ["rclone", "mkdir", f"cryptLocal:{subfolder_label}"],
-            check=True
-        )
-        logger.info(f"[OK] rclone mkdir cryptLocal:{subfolder_label}")
-    except subprocess.CalledProcessError as e:
-        logger.error(f"rclone mkdir failed: {e}")
-        return None
-
-    # 3) local_folder -> cryptLocal:subfolder_label にコピー
-    #    --create-empty-src-dirs: 空フォルダ構造も含めてコピー
-    try:
-        subprocess.run(
-            [
-                "rclone", "copy",
-                local_folder,
-                f"cryptLocal:{subfolder_label}",
-                "--create-empty-src-dirs"
-            ],
-            check=True
-        )
-        logger.info(f"[OK] rclone copy {local_folder} => cryptLocal:{subfolder_label}")
-    except subprocess.CalledProcessError as e:
-        logger.error(f"rclone copy failed: {e}")
-        return None
-
-    # cryptLocal:{subfolder_label} => 実際には /home/user/app/encrypted/<ランダム文字列> のフォルダになる
-    # それを差分検知で特定
-    before_dirs = set(os.listdir(encrypted_base_dir))
-    # mkdir/copy 直後に何らかのズレがあるかもしれないので再度 mkdir しなくてOK
-    # すでに上でやってるため、ここでやるなら:
-    # time.sleep(1) などで間を置いてみる手もある
-
-    after_dirs = set(os.listdir(encrypted_base_dir))
-    diff = after_dirs - before_dirs
-    #   もし diff が空なら既にあるフォルダに上書きコピーされた可能性
-    if not diff:
-        logger.error("[ERROR] No new directory appeared in ./encrypted after rclone copy.")
-        return None
-    if len(diff) > 1:
-        logger.warning(f"[WARN] Multiple new directories found: {diff}, picking the first one.")
-    enc_folder_name = diff.pop()
-    enc_folder_path = os.path.join(encrypted_base_dir, enc_folder_name)
-
-    if not os.path.isdir(enc_folder_path):
-        logger.error(f"[ERROR] {enc_folder_path} is not a directory.")
-        return None
-
-    # 4) アップロード (フォルダとして)
-    try:
-        self.upload_folder(enc_folder_path, path_in_repo=subfolder_label)
-        logger.info(f"Uploaded encrypted folder: {enc_folder_path}")
-    except Exception as e:
-        logger.error(f"Failed to upload encrypted folder {enc_folder_path}: {e}")
-
-    # 5) ローカル削除
-    try:
-        shutil.rmtree(local_folder)
-        shutil.rmtree(enc_folder_path)
-        logger.info(f"Removed local folder: {local_folder} and {enc_folder_path}")
-    except Exception as e:
-        logger.error(f"Failed to remove local folders: {e}")
-
-    # 6) 論理上のフォルダ名（rclone上の名称）を返す
-    return subfolder_label
+            logger.error(f"Error saving HTML content from {model_page_url}: {e}")
+
+    def save_model_info_json(self, model_info: dict, folder: str):
+        """モデル情報をJSONファイルとして保存."""
+        info_path = os.path.join(folder, "model_info.json")
+        try:
+            with open(info_path, 'w', encoding='utf-8') as f:
+                json.dump(model_info, f, indent=2)
+            logger.info(f"Saved model_info.json: {info_path}")
+        except Exception as e:
+            logger.error(f"Failed to save model info JSON: {e}")
+    # =========================================================================
+    # ここが重要:
+    #  - 最新バージョンはまとめて folder_name にダウンロード (一度に暗号化アップロード)
+    #  - 古いバージョンは1つずつダウンロード→暗号化→アップロード→削除 でストレージを節約
+    # =========================================================================
+    def download_and_process_versions(self, model_versions: list, folder: str):
+        """最新バージョンをまとめてダウンロード、old_versionsは1つずつアップして削除。"""
+
+        # 1) 最新バージョン (インデックス0) のファイルを folder にダウンロード
+        latest_version = model_versions[0]
+        logger.info(f"Processing latest version: {latest_version.get('name','(NoName)')}")
+
+        for file_info in latest_version.get("files", []):
+            download_url = file_info["downloadUrl"]
+            file_name = file_info["name"]
+            login_detected_count = 0
+
+            while login_detected_count < 5:
+                local_path = self.download_file(download_url, folder, file_name)
+                if local_path and "login" in os.listdir(folder):
+                    login_detected_count += 1
+                    os.remove(os.path.join(folder, "login"))
+                    logger.warning(f"'login' file found, retrying {file_name} ({login_detected_count}/5)")
+                else:
+                    break
+
+            if login_detected_count >= 5:
+                dummy_file_path = os.path.join(folder, f"{file_name}.download_failed")
+                try:
+                    with open(dummy_file_path, "w") as f:
+                        f.write("Download failed after 5 attempts.")
+                    logger.error(f"Failed to download {file_name}. Dummy file created: {dummy_file_path}")
+                except Exception as e:
+                    logger.error(f"Failed to create dummy file for {file_name}: {e}")
+
+        # 2) 古いバージョンがあれば 1つずつダウンロード→暗号化アップロード→削除
+        if len(model_versions) > 1:
+            old_versions_folder = os.path.join(folder, "old_versions")
+            os.makedirs(old_versions_folder, exist_ok=True)
+
+            for version in model_versions[1:]:
+                logger.info(f"Processing older version: {version.get('name','(NoName)')}")
+                for file_info in version.get("files", []):
+                    file_name = file_info["name"]
+                    download_url = file_info["downloadUrl"]
+                    login_detected_count = 0
+
+                    while login_detected_count < 5:
+                        local_path = self.download_file(download_url, old_versions_folder, file_name)
+                        if local_path and "login" in os.listdir(old_versions_folder):
+                            login_detected_count += 1
+                            os.remove(os.path.join(old_versions_folder, "login"))
+                            logger.warning(f"'login' file found, retrying {file_name} ({login_detected_count}/5)")
+                        else:
+                            break
+
+                    if login_detected_count >= 5:
+                        dummy_file_path = os.path.join(old_versions_folder, f"{file_name}.download_failed")
+                        try:
+                            with open(dummy_file_path, "w") as f:
+                                f.write("Download failed after 5 attempts.")
+                            logger.error(f"Failed to download {file_name}. Dummy file: {dummy_file_path}")
+                        except Exception as e:
+                            logger.error(f"Failed to create dummy file for {file_name}: {e}")
+                        continue  # 次のファイルへ
+
+                    # ===== ダウンロード成功した古いバージョンファイルを暗号化アップロード =====
+                    # フォルダごと暗号化でもいいですが、大容量を避けるためファイル単位で暗号化する例
+                    # ここでは "encrypt_and_upload_folder" の代わりに「encrypt_and_upload_file」するなど
+                    #   あるいはフォルダごとでもOK
+                    single_file_folder = os.path.join(old_versions_folder, "temp_single")
+                    os.makedirs(single_file_folder, exist_ok=True)
+                    try:
+                        # 移動して「このファイルだけ」が入ったフォルダを作る
+                        single_file_path = shutil.move(local_path, os.path.join(single_file_folder, file_name))
+                        # 暗号化アップロード
+                        self.encrypt_and_upload_folder(single_file_folder)
+                    except Exception as e:
+                        logger.error(f"Failed to encrypt/upload old version file: {e}")
+                    finally:
+                        # single_file_folder削除（encrypt_and_upload_folderで消えるはず）
+                        if os.path.exists(single_file_folder):
+                            shutil.rmtree(single_file_folder)
+                            logger.info(f"Removed temp_single folder {single_file_folder}")
+
+    def encrypt_and_upload_folder(self, local_folder: str) -> Optional[str]:
+        """
+        1. /home/user/app/encrypted 配下の古いファイルやフォルダを消去（オプション）
+        2. rclone mkdir cryptLocal:subfolder_label  (空ディレクトリを必ず作る)
+        3. rclone copy local_folder => cryptLocal:subfolder_label
+        → /home/user/app/encrypted/<暗号フォルダ>/ にコピー
+        4. そのフォルダを self.upload_folder() (＝HFにアップロード)
+        5. ローカル(平文フォルダ & 暗号フォルダ)削除
+        6. 最後に subfolder_label（論理上のフォルダ名）を return
+        """
+        if not os.path.exists(local_folder):
+            logger.error(f"encrypt_and_upload_folder: folder not found: {local_folder}")
+            return None
+
+        encrypted_base_dir = os.path.join(os.getcwd(), "encrypted")
+        os.makedirs(encrypted_base_dir, exist_ok=True)
+
+        # 1) 古い暗号ファイルやフォルダを削除
+        for item in os.listdir(encrypted_base_dir):
+            item_path = os.path.join(encrypted_base_dir, item)
+            try:
+                if os.path.isfile(item_path) or os.path.islink(item_path):
+                    os.remove(item_path)
+                else:
+                    shutil.rmtree(item_path)
+                logger.info(f"[CLEANUP] Removed old encrypted item: {item_path}")
+            except Exception as e:
+                logger.warning(f"[CLEANUP] Failed to remove {item_path}: {e}")
+
+        # 2) まず mkdir で「暗号リモート上に空ディレクトリ」を作る
+        subfolder_label = "enc_" + str(uuid.uuid4())[:8]
+        try:
+            subprocess.run(
+                ["rclone", "mkdir", f"cryptLocal:{subfolder_label}"],
+                check=True
+            )
+            logger.info(f"[OK] rclone mkdir cryptLocal:{subfolder_label}")
+        except subprocess.CalledProcessError as e:
+            logger.error(f"rclone mkdir failed: {e}")
+            return None
+
+        # 3) local_folder -> cryptLocal:subfolder_label にコピー
+        #    --create-empty-src-dirs: 空フォルダ構造も含めてコピー
+        try:
+            subprocess.run(
+                [
+                    "rclone", "copy",
+                    local_folder,
+                    f"cryptLocal:{subfolder_label}",
+                    "--create-empty-src-dirs"
+                ],
+                check=True
+            )
+            logger.info(f"[OK] rclone copy {local_folder} => cryptLocal:{subfolder_label}")
+        except subprocess.CalledProcessError as e:
+            logger.error(f"rclone copy failed: {e}")
+            return None
+
+        # cryptLocal:{subfolder_label} => 実際には /home/user/app/encrypted/<ランダム文字列> のフォルダになる
+        # それを差分検知で特定
+        before_dirs = set(os.listdir(encrypted_base_dir))
+        # mkdir/copy 直後に何らかのズレがあるかもしれないので再度 mkdir しなくてOK
+        # すでに上でやってるため、ここでやるなら:
+        # time.sleep(1) などで間を置いてみる手もある
+
+        after_dirs = set(os.listdir(encrypted_base_dir))
+        diff = after_dirs - before_dirs
+        #   もし diff が空なら既にあるフォルダに上書きコピーされた可能性
+        if not diff:
+            logger.error("[ERROR] No new directory appeared in ./encrypted after rclone copy.")
+            return None
+        if len(diff) > 1:
+            logger.warning(f"[WARN] Multiple new directories found: {diff}, picking the first one.")
+        enc_folder_name = diff.pop()
+        enc_folder_path = os.path.join(encrypted_base_dir, enc_folder_name)
+
+        if not os.path.isdir(enc_folder_path):
+            logger.error(f"[ERROR] {enc_folder_path} is not a directory.")
+            return None
+
+        # 4) アップロード (フォルダとして)
+        try:
+            self.upload_folder(enc_folder_path, path_in_repo=subfolder_label)
+            logger.info(f"Uploaded encrypted folder: {enc_folder_path}")
+        except Exception as e:
+            logger.error(f"Failed to upload encrypted folder {enc_folder_path}: {e}")
+
+        # 5) ローカル削除
+        try:
+            shutil.rmtree(local_folder)
+            shutil.rmtree(enc_folder_path)
+            logger.info(f"Removed local folder: {local_folder} and {enc_folder_path}")
+        except Exception as e:
+            logger.error(f"Failed to remove local folders: {e}")
+
+        # 6) 論理上のフォルダ名（rclone上の名称）を返す
+        return subfolder_label
+
+    def upload_file(self, file_path: str, repo_id: Optional[str] = None, path_in_repo: Optional[str] = None):
+        if repo_id is None:
+            repo_id = self.repo_ids['current']
+        if path_in_repo is None:
+            path_in_repo = os.path.basename(file_path)
+        
+        max_retries = 5
+        attempt = 0
+        while attempt < max_retries:
+            try:
+                self.api.upload_file(
+                    path_or_fileobj=file_path,
+                    repo_id=repo_id,
+                    path_in_repo=path_in_repo
+                )
+                logger.info(f"Uploaded file: {file_path} to {repo_id} at {path_in_repo}")
+                return
+            except Exception as e:
+                attempt += 1
+                error_message = str(e)
+                if "over the limit of 100000 files" in error_message:
+                    logger.warning("File limit exceeded, creating a new repo.")
+                    self.repo_ids['current'] = self.increment_repo_name(self.repo_ids['current'])
+                    self.api.create_repo(repo_id=self.repo_ids['current'], private=True)
+                    attempt = 0
+                    continue
+                elif "you can retry this action in about 1 hour" in error_message:
+                    logger.warning("Rate limit hit. Waiting 1 hour...")
+                    time.sleep(3600)
+                    attempt -= 1
+                else:
+                    if attempt < max_retries:
+                        logger.warning(f"Failed to upload {file_path}, retry {attempt}/{max_retries}")
+                    else:
+                        logger.error(f"Failed after {max_retries} attempts: {e}")
+                        raise
+
+    def upload_file(self, file_path: str, repo_id: Optional[str] = None, path_in_repo: Optional[str] = None):
+        if repo_id is None:
+            repo_id = self.repo_ids['current']
+        if path_in_repo is None:
+            path_in_repo = os.path.basename(file_path)
+        
+        max_retries = 5
+        attempt = 0
+        while attempt < max_retries:
+            try:
+                self.api.upload_file(
+                    path_or_fileobj=file_path,
+                    repo_id=repo_id,
+                    path_in_repo=path_in_repo
+                )
+                logger.info(f"Uploaded file: {file_path} to {repo_id} at {path_in_repo}")
+                return
+            except Exception as e:
+                attempt += 1
+                error_message = str(e)
+                if "over the limit of 100000 files" in error_message:
+                    logger.warning("File limit exceeded, creating a new repo.")
+                    self.repo_ids['current'] = self.increment_repo_name(self.repo_ids['current'])
+                    self.api.create_repo(repo_id=self.repo_ids['current'], private=True)
+                    attempt = 0
+                    continue
+                elif "you can retry this action in about 1 hour" in error_message:
+                    logger.warning("Rate limit hit. Waiting 1 hour...")
+                    time.sleep(3600)
+                    attempt -= 1
+                else:
+                    if attempt < max_retries:
+                        logger.warning(f"Failed to upload {file_path}, retry {attempt}/{max_retries}")
+                    else:
+                        logger.error(f"Failed after {max_retries} attempts: {e}")
+                        raise
+
+    def upload_folder(self, folder_path: str, path_in_repo: Optional[str] = None):
+        """
+        フォルダを Hugging Face リポジトリに一括アップロード
+        """
+        if path_in_repo is None:
+            path_in_repo = os.path.basename(folder_path)
+
+        max_retries = 5
+        attempt = 0
+        while attempt < max_retries:
+            try:
+                self.api.upload_folder(
+                    folder_path=folder_path,
+                    repo_id=self.repo_ids['current'],
+                    path_in_repo=path_in_repo
+                )
+                logger.info(f"Uploaded folder: {folder_path} to {self.repo_ids['current']} at {path_in_repo}")
+                return
+            except Exception as e:
+                attempt += 1
+                error_message = str(e)
+                if "over the limit of 100000 files" in error_message:
+                    logger.warning("File limit exceeded, creating a new repo.")
+                    self.repo_ids['current'] = self.increment_repo_name(self.repo_ids['current'])
+                    self.api.create_repo(repo_id=self.repo_ids['current'], private=True)
+                    attempt = 0
+                    continue
+                elif "you can retry this action in about 1 hour" in error_message:
+                    logger.warning("Rate limit hit. Waiting 1 hour...")
+                    time.sleep(3600)
+                    attempt -= 1
+                else:
+                    if attempt < max_retries:
+                        logger.warning(f"Failed to upload folder {folder_path}, retry {attempt}/{max_retries}")
+                    else:
+                        logger.error(f"Failed after {max_retries} attempts: {e}")
+                        raise
+
+    @staticmethod
+    def increment_repo_name(repo_id: str) -> str:
+        """リポジトリ名の末尾の数字をインクリメントする。"""
+        match = re.search(r'(\d+)$', repo_id)
+        if match:
+            number = int(match.group(1)) + 1
+            new_repo_id = re.sub(r'\d+$', str(number), repo_id)
+        else:
+            new_repo_id = f"{repo_id}1"
+        return new_repo_id
+
+    def read_model_list(self) -> dict:
+        """モデルリストを読み込む。"""
+        model_list = {}
+        try:
+            with open(self.config.LIST_FILE, "r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if line:
+                        parts = line.split(": ", 1)
+                        if len(parts) == 2:
+                            modelpage_name, model_hf_url = parts
+                            model_list[model_hf_url] = modelpage_name
+        except Exception as e:
+            logger.error(f"Failed to read model list: {e}")
+        return model_list
+
+    def get_repo_info(self, repo_id):
+        """リポジトリの情報を取得する。"""
+        try:
+            repo_info = self.api.repo_info(repo_id=repo_id, files_metadata=True)
+            file_paths = [sibling.rfilename for sibling in repo_info.siblings]
+            return file_paths
+        except Exception as e:
+            logger.error(f"Failed to get repo info for {repo_id}: {e}")
+            return []
+
+    def process_model(self, model_url: str):
+        """1つのモデルをダウンロードしてフォルダ丸ごと暗号化＆アップロードする."""
+        try:
+            # model_idを取得
+            model_id = model_url.rstrip("/").split("/")[-1]
+
+            # モデル情報を取得
+            model_info = self.get_model_info(model_id)
+            if not model_info or "modelVersions" not in model_info:
+                logger.error(f"No valid model info for ID {model_id}. Skipping.")
+                return
+
+            # バージョン一覧
+            versions = model_info["modelVersions"]
+            if not versions:
+                logger.warning(f"No modelVersions found for ID {model_id}.")
+                return
+
+            # ベースとなるフォルダ名
+            folder_name = model_info.get("name", "UnknownModel")
+            folder_name = re.sub(r'[\\/*?:"<>|]', '_', folder_name)
+            folder_name += "_" + str(uuid.uuid4())[:8]
+            os.makedirs(folder_name, exist_ok=True)
+
+            # (A) 最新バージョンファイル + (B) 画像 + (C) HTML + model_info.json
+            #    → 全部 folder_name に置く
+            #    → 後でまとめて encrypt_and_upload_folder
+            self.download_and_process_versions(versions, folder_name)
+            self.download_images(versions, folder_name)
+            self.save_html_content(f"{self.config.URLS['modelPage']}{model_id}", folder_name)
+            self.save_model_info_json(model_info, folder_name)
+
+            # ↑ この時点で「最新バージョン」の大ファイルが folder_name に残っている
+            #   しかし old_versions は既に1つずつ暗号化→削除済み
+
+            # まとめて暗号化 & アップロード
+            enc_subfolder = self.encrypt_and_upload_folder(folder_name)
+            if enc_subfolder is None:
+                enc_subfolder = "[ENCRYPT_FAILED]"
+
+            hf_enc_url = f"https://huggingface.co/{self.repo_ids['current']}/tree/main/{enc_subfolder}"
+            with open(self.config.LIST_FILE, "a", encoding="utf-8") as f:
+                f.write(f"{model_info.get('name', 'UnnamedModel')} (ID:{model_id}): {hf_enc_url}\n")
+
+        except Exception as e:
+            logger.error(f"Error in process_model ({model_url}): {e}")
+
+    async def crawl(self):
+        """モデルを定期的にチェックし、更新を行う。"""
+        while True:
+            try:
+                login(token=self.config.HUGGINGFACE_API_KEY, add_to_git_credential=True)
+
+                # model_list.logを最新化
+                model_list_path = hf_hub_download(
+                    repo_id=self.repo_ids['model_list'],
+                    filename=self.config.LIST_FILE
+                )
+                shutil.copyfile(model_list_path, f"./{self.config.LIST_FILE}")
+
+                # ログファイルを最新化
+                local_file_path = hf_hub_download(
+                    repo_id=self.repo_ids["log"],
+                    filename=self.config.LOG_FILE
+                )
+                shutil.copyfile(local_file_path, f"./{self.config.LOG_FILE}")
+
+                # ログを読み込み
+                with open(self.config.LOG_FILE, "r", encoding="utf-8") as file:
+                    lines = file.read().splitlines()
+                    old_models = json.loads(lines[0]) if len(lines) > 0 else []
+                    self.repo_ids["current"] = lines[1] if len(lines) > 1 else ""
+
+                # 新着モデル確認
+                r = requests.get(self.config.URLS["latest"], headers=self.config.HEADERS)
+                r.raise_for_status()
+                latest_models = r.json().get("items", [])
+                latest_model_ids = [m["id"] for m in latest_models if "id" in m]
+
+                new_models = list(set(latest_model_ids) - set(old_models))
+                if new_models:
+                    logger.info(f"New model IDs found: {new_models}")
+                    model_id = new_models[0]
+
+                    for attempt in range(1, 6):
+                        try:
+                            self.process_model(self.config.URLS["modelId"] + str(model_id))
+                            break
+                        except Exception as e:
+                            logger.error(f"Failed to process model {model_id} (attempt {attempt}/5): {e}")
+                            if attempt == 5:
+                                logger.error(f"Skipping model {model_id} after 5 failures.")
+                            else:
+                                await asyncio.sleep(2)
+
+                    # 新モデルをold_modelsに追加し、ログを更新
+                    old_models.append(model_id)
+                    with open(self.config.LOG_FILE, "w", encoding="utf-8") as f:
+                        f.write(json.dumps(old_models) + "\n")
+                        f.write(f"{self.repo_ids['current']}\n")
+                    logger.info(f"Updated log with new model ID: {model_id}")
+
+                    # ログファイル & model_list.logをアップロード
+                    self.upload_file(
+                        file_path=self.config.LOG_FILE,
+                        repo_id=self.repo_ids["log"],
+                        path_in_repo=self.config.LOG_FILE
+                    )
+                    self.upload_file(
+                        file_path=self.config.LIST_FILE,
+                        repo_id=self.repo_ids["model_list"],
+                        path_in_repo=self.config.LIST_FILE
+                    )
+                else:
+                    # 新着なし → ログを最新化してアップロードだけして待機
+                    with open(self.config.LOG_FILE, "w", encoding="utf-8") as f:
+                        f.write(json.dumps(latest_model_ids) + "\n")
+                        f.write(f"{self.repo_ids['current']}\n")
+                    logger.info(f"No new models. Updated log: {self.config.LOG_FILE}")
+                    self.upload_file(
+                        file_path=self.config.LOG_FILE,
+                        repo_id=self.repo_ids["log"],
+                        path_in_repo=self.config.LOG_FILE
+                    )
+                    logger.info("Uploaded log file.")
+                    await asyncio.sleep(60)
+                    continue
+
+            except Exception as e:
+                logger.error(f"Error in crawl loop: {e}")
+                await asyncio.sleep(300)
+
+
+# FastAPIアプリケーション
+config = Config()
+crawler = CivitAICrawler(config)
+app = crawler.app