Upload main.py

main.py

@@ -1,4 +1,5 @@
 import asyncio
+import base64
 import datetime
 import json
 import logging
@@ -13,20 +14,23 @@ import requests
 from bs4 import BeautifulSoup
 from fake_useragent import UserAgent
 from fastapi import FastAPI
-from huggingface_hub import HfApi, hf_hub_download, login
+from huggingface_hub import HfApi, create_repo, hf_hub_download, login
 
+# ロギングの設定
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+
 class Config:
+    """設定用のクラス"""
     HUGGINGFACE_API_KEY = os.environ["HUGGINGFACE_API_KEY"]
     CIVITAI_API_TOKEN = os.environ["CIVITAI_API_TOKEN"]
     LOG_FILE = "civitai_backup.log"
     LIST_FILE = "model_list.log"
     REPO_IDS = {
-        "log": "ttttdiva/CivitAI_log_test",       
+        "log": "ttttdiva/CivitAI_log_test",
         "model_list": "ttttdiva/CivitAI_model_info_test",
-        "current": ""  
+        "current": ""
     }
     URLS = {
         "latest": "https://civitai.com/api/v1/models?sort=Newest",
@@ -43,281 +47,535 @@ class Config:
         "Content-Type": "application/json"
     }
 
+    # ===== rclone 用の追加設定 =====
+    # （環境変数 RCLONE_CONF_BASE64 に rclone.conf をbase64エンコードした文字列を設定しておく想定）
+    RCLONE_CONF_BASE64 = os.environ.get("RCLONE_CONF_BASE64", "")
+    # 暗号化されたファイルが出力されるローカルディレクトリ（cryptLocal: の実体）
+    ENCRYPTED_DIR = "/home/user/app/encrypted"
+
+
 class CivitAICrawler:
+    """CivitAIからモデルをダウンロードし、Hugging Faceにアップロードするクラス（rcloneで暗号化対応版）"""
+
     def __init__(self, config: Config):
         self.config = config
         self.api = HfApi()
         self.app = FastAPI()
         self.repo_ids = self.config.REPO_IDS.copy()
         self.jst = self.config.JST
+
+        # rclone のセットアップ
         self.setup_rclone_conf()
-        self.setup_routes()
 
-    def setup_rclone_conf(self):
-        import base64
-        rclone_b64 = os.environ.get("RCLONE_CONF_BASE64", "")
-        if rclone_b64:
-            conf_dir = ".rclone_config"
-            os.makedirs(conf_dir, exist_ok=True)
-            conf_path = os.path.join(conf_dir, "rclone.conf")
-            with open(conf_path, "wb") as f:
-                f.write(base64.b64decode(rclone_b64))
-            os.environ["RCLONE_CONFIG"] = conf_path
-            logger.info(f"[OK] Created rclone.conf => {conf_path}")
-        else:
-            logger.warning("[WARN] RCLONE_CONF_BASE64 is empty. rclone may fail.")
+        self.setup_routes()
 
     def setup_routes(self):
+        """FastAPIのルーティングを設定する。"""
+
         @self.app.get("/")
         def read_root():
             now = str(datetime.datetime.now(self.jst))
-            return {
-                "description": f"CivitAI crawler. Time: {now}",
-                "repo_current": self.repo_ids["current"]
-            }
+            description = f"""
+            CivitAIを定期的に周回し新規モデルを {self.repo_ids['current']} にバックアップするspaceです。
+            モデルページ名とバックアップURLの紐づきはhttps://huggingface.co/{self.repo_ids['model_list']}/blob/main/model_list.logからどうぞ
+            たまに覗いてもらえると動き続けると思います。
+            再起動が必要になっている場合はRestartボタンを押してもらえると助かります。
+            Status: {now} + currently running :D
+            """
+            return description
 
         @self.app.on_event("startup")
         async def startup_event():
             asyncio.create_task(self.crawl())
 
-    def download_file(self, url: str, dest_folder: str, filename: str) -> Optional[str]:
-        os.makedirs(dest_folder, exist_ok=True)
+    # =============================
+    # rclone 周りのヘルパー関数
+    # =============================
+    def setup_rclone_conf(self):
+        """環境変数 RCLONE_CONF_BASE64 から rclone.conf を生成し、RCLONE_CONFIG 環境変数を設定"""
+        if not self.config.RCLONE_CONF_BASE64:
+            logger.warning("[WARN] RCLONE_CONF_BASE64 is empty. rclone may fail.")
+            return
+
+        os.makedirs(".rclone_config", exist_ok=True)
+        conf_path = os.path.join(".rclone_config", "rclone.conf")
+        with open(conf_path, "wb") as f:
+            f.write(base64.b64decode(self.config.RCLONE_CONF_BASE64))
+
+        os.environ["RCLONE_CONFIG"] = conf_path
+        logger.info(f"[INFO] rclone.conf created at: {conf_path}")
+
+    def encrypt_with_rclone(self, local_path: str, is_file: bool = True):
+        """
+        指定ファイル or フォルダを cryptLocal: へコピーし、暗号化ファイルを self.config.ENCRYPTED_DIR に生成する。
+        rclone copy の引数にフォルダパスやファイルパスを指定して利用可能。
+        """
+        if not os.path.exists(local_path):
+            raise FileNotFoundError(f"[ERROR] Local path not found: {local_path}")
+
+        # 事前に暗号先ディレクトリをクリーンアップ（不要なら削除する）
+        if os.path.isdir(self.config.ENCRYPTED_DIR):
+            shutil.rmtree(self.config.ENCRYPTED_DIR, ignore_errors=True)
+
+        # rclone コマンドの実行
+        cmd = ["rclone", "copy", local_path, "cryptLocal:", "-v"]
+        logger.info(f"[INFO] Running: {' '.join(cmd)}")
+        subprocess.run(cmd, check=True)
+        logger.info(f"[OK] rclone copy {local_path} => cryptLocal:")
+
+        if not os.path.isdir(self.config.ENCRYPTED_DIR):
+            raise FileNotFoundError(
+                f"[ERROR] {self.config.ENCRYPTED_DIR} not found. Check your rclone config."
+            )
+
+    def upload_encrypted_files(self, repo_id: str, path_in_repo: str = None):
+        """
+        self.config.ENCRYPTED_DIR にある暗号化済みファイルをまとめて Hugging Face にアップロードする。
+        もとがフォルダの場合はサブディレクトリも含めて再帰的にアップロードする。
+        """
+        if not path_in_repo:
+            path_in_repo = ""
+
+        max_retries = 5
+
+        # 再帰的に walk
+        for root, dirs, files in os.walk(self.config.ENCRYPTED_DIR):
+            for fn in files:
+                encrypted_file_path = os.path.join(root, fn)
+                if not os.path.isfile(encrypted_file_path):
+                    continue
+
+                # 元ディレクトリ相対のパスを生成
+                relative_path = os.path.relpath(encrypted_file_path, self.config.ENCRYPTED_DIR)
+                # Hugging Face 上に保存するときのフルパス
+                upload_path_in_repo = os.path.join(path_in_repo, relative_path)
+
+                # HFへのアップロードを試行 (over the limitなどの例外をリトライする)
+                attempt = 0
+                while attempt < max_retries:
+                    try:
+                        self.api.upload_file(
+                            path_or_fileobj=encrypted_file_path,
+                            repo_id=repo_id,
+                            path_in_repo=upload_path_in_repo
+                        )
+                        logger.info(f"[OK] Uploaded {encrypted_file_path} to {repo_id}/{upload_path_in_repo}")
+                        break  # 成功したらループ抜け
+                    except Exception as e:
+                        attempt += 1
+                        error_message = str(e)
+                        if "over the limit of 100000 files" in error_message:
+                            logger.warning("Repository file limit exceeded, creating a new repository.")
+                            self.repo_ids['current'] = self.increment_repo_name(self.repo_ids['current'])
+                            self.api.create_repo(repo_id=self.repo_ids['current'], private=True)
+                            # リポジトリを変えたので attempt をリセット
+                            attempt = 0
+                            repo_id = self.repo_ids['current']
+                            continue
+                        elif "you can retry this action in about 1 hour" in error_message:
+                            logger.warning("Encountered 'retry in 1 hour' error. Waiting 1 hour before retrying...")
+                            time.sleep(3600)
+                            attempt -= 1  # この場合はリトライ回数をカウントしない
+                        else:
+                            if attempt < max_retries:
+                                logger.warning(f"Failed to upload file {encrypted_file_path}, retrying... {attempt}/{max_retries}")
+                            else:
+                                logger.error(f"Failed to upload file after {max_retries} attempts: {encrypted_file_path}")
+                                raise
+
+    # =============================
+    # ここから既存処理
+    # =============================
+
+    @staticmethod
+    def get_filename_from_cd(content_disposition: Optional[str], default_name: str) -> str:
+        """Content-Dispositionヘッダーからファイル名を取得する。"""
+        if content_disposition:
+            parts = content_disposition.split(';')
+            for part in parts:
+                if "filename=" in part:
+                    return part.split("=")[1].strip().strip('"')
+        return default_name
+
+    def download_file(self, url: str, destination_folder: str, default_name: str):
+        """指定されたURLからファイルをダウンロードし、指定されたフォルダに保存する。"""
         try:
-            r = requests.get(url, headers=self.config.HEADERS, stream=True)
-            r.raise_for_status()
+            response = requests.get(url, headers=self.config.HEADERS, stream=True)
+            response.raise_for_status()
         except requests.RequestException as e:
-            logger.error(f"[ERR] download_file: {e}")
-            return None
+            logger.error(f"Failed to download file from {url}: {e}")
+            return
 
-        file_path = os.path.join(dest_folder, filename)
-        with open(file_path, 'wb') as f:
-            for chunk in r.iter_content(chunk_size=8192):
-                f.write(chunk)
-        logger.info(f"[OK] Downloaded => {file_path}")
-        return file_path
+        filename = self.get_filename_from_cd(response.headers.get('content-disposition'), default_name)
+        file_path = os.path.join(destination_folder, filename)
 
-    def upload_file(self, file_path: str, repo_id: Optional[str]=None, path_in_repo: Optional[str]=None):
-        if repo_id is None:
-            repo_id = self.repo_ids["current"]
-        if path_in_repo is None:
-            path_in_repo = os.path.basename(file_path)
+        with open(file_path, 'wb') as file:
+            for chunk in response.iter_content(chunk_size=8192):
+                file.write(chunk)
+        logger.info(f"Download completed: {file_path}")
 
+    def get_model_info(self, model_id: str) -> dict:
+        """モデルの情報を取得する。"""
         try:
-            self.api.upload_file(
-                path_or_fileobj=file_path,
-                repo_id=repo_id,
-                path_in_repo=path_in_repo
-            )
-            logger.info(f"[OK] Uploaded file => {repo_id}:{path_in_repo}")
-        except Exception as e:
-            logger.error(f"[ERR] upload_file: {e}")
+            response = requests.get(self.config.URLS["modelId"] + str(model_id), headers=self.config.HEADERS)
+            response.raise_for_status()
+            return response.json()
+        except requests.RequestException as e:
+            logger.error(f"Failed to retrieve model info for ID {model_id}: {e}")
+
+    def download_model(self, model_versions: list, folder: str, existing_old_version_files: list = []):
+        """モデルのバージョンをダウンロードする。"""
+        latest_version = model_versions[0]
+        latest_files = latest_version["files"]
+        for file_info in latest_files:
+            download_url = file_info["downloadUrl"]
+            file_name = file_info["name"]
+            login_detected_count = 0
+
+            while login_detected_count < 5:
+                try:
+                    self.download_file(download_url, folder, file_name)
+                except Exception as e:
+                    logger.error(f"Exception occurred while downloading {file_name}: {e}")
+                    login_detected_count += 1
+                    continue
 
-    def upload_folder(self, folder_path: str, path_in_repo: Optional[str] = None):
-        if path_in_repo is None:
-            path_in_repo = os.path.basename(folder_path)
+                if "login" in os.listdir(folder):
+                    login_detected_count += 1
+                    logger.warning(f"'login' file found. Will try again. ({login_detected_count}/5)")
+                    os.remove(os.path.join(folder, "login"))
+                else:
+                    logger.info(f"Successfully downloaded {file_name}")
+                    break
+
+            if login_detected_count >= 5:
+                dummy_file_name = f"{file_name}.download_failed"
+                dummy_file_path = os.path.join(folder, dummy_file_name)
+                try:
+                    with open(dummy_file_path, "w") as f:
+                        f.write("Download failed after 5 attempts.")
+                    logger.error(f"Failed to download {file_name}. Created dummy file {dummy_file_name}. URL: {download_url}")
+                except Exception as e:
+                    logger.error(f"Failed to create dummy file for {file_name}: {e}")
+
+        # 古いバージョンのダウンロード
+        if len(model_versions) > 1:
+            old_versions_folder = os.path.join(folder, "old_versions")
+            os.makedirs(old_versions_folder, exist_ok=True)
+            for version in model_versions[1:]:
+                for file_info in version["files"]:
+                    file_name = file_info["name"]
+                    if file_name in existing_old_version_files:
+                        logger.info(f"Skipping download of existing old version file: {file_name}")
+                        continue
+                    download_url = file_info["downloadUrl"]
+                    local_file_path = os.path.join(old_versions_folder, file_name)
+                    login_detected_count = 0
+
+                    while login_detected_count < 5:
+                        try:
+                            self.download_file(download_url, old_versions_folder, file_name)
+                        except Exception as e:
+                            logger.error(f"Exception occurred while downloading {file_name}: {e}")
+                            login_detected_count += 1
+                            continue
+
+                        if "login" in os.listdir(old_versions_folder):
+                            login_detected_count += 1
+                            logger.warning(f"'login' file found while downloading {file_name}. Will try again. ({login_detected_count}/5)")
+                            os.remove(os.path.join(old_versions_folder, "login"))
+                        else:
+                            logger.info(f"Successfully downloaded {file_name}")
+                            break
 
-        try:
-            self.api.upload_folder(
-                folder_path=folder_path,
-                repo_id=self.repo_ids["current"],
-                path_in_repo=path_in_repo
-            )
-            logger.info(f"[OK] uploaded folder => {folder_path} => {self.repo_ids['current']}:{path_in_repo}")
-        except Exception as e:
-            logger.error(f"[ERR] upload_folder: {e}")
+                    if login_detected_count >= 5:
+                        dummy_file_name = f"{file_name}.download_failed"
+                        dummy_file_path = os.path.join(old_versions_folder, dummy_file_name)
+                        try:
+                            with open(dummy_file_path, "w") as f:
+                                f.write("Download failed after 5 attempts.")
+                            logger.error(f"Failed to download {file_name}. Created dummy file {dummy_file_name}. URL: {download_url}")
+                        except Exception as e:
+                            logger.error(f"Failed to create dummy file for {file_name}: {e}")
+                        continue
 
-    def encrypt_and_upload_folder(self, local_folder: str) -> Optional[str]:
-        """local_folder -> cryptLocal: => encrypted/??? => upload_folder => cleanup"""
-        if not os.path.isdir(local_folder):
-            logger.error(f"[ERR] {local_folder} is not a directory.")
-            return None
+                    # ===== 旧コード: 直接アップロードしていた箇所を削除して、rclone暗号化＆アップロードに変更する場合は呼び出さない =====
+                    # 旧来は self.upload_file(...) していたが、このタイミングでアップロードしたくない場合は消すかコメントアウト
+                    # self.upload_file(local_file_path, path_in_repo=...)
+                    # os.remove(local_file_path)
 
-        encrypted_dir = os.path.join(os.getcwd(), "encrypted")
-        os.makedirs(encrypted_dir, exist_ok=True)
+    def download_images(self, model_versions: list, folder: str):
+        """モデルの画像をダウンロードし、指定されたフォルダに保存する。"""
+        images_folder = os.path.join(folder, "images")
+        os.makedirs(images_folder, exist_ok=True)
 
-        before = set(os.listdir(encrypted_dir))
-        cmd = ["rclone", "copy", local_folder, "cryptLocal:", "--create-empty-src-dirs"]
-        logger.info(f"[CMD] {' '.join(cmd)}")
+        images = []
+        for version in model_versions:
+            for img in version.get("images", []):
+                image_url = img["url"]
+                images.append(image_url)
+
+        for image_url in images:
+            image_name = image_url.split("/")[-1]
+            try:
+                response = requests.get(image_url)
+                response.raise_for_status()
+                with open(os.path.join(images_folder, f"{image_name}.png"), "wb") as file:
+                    file.write(response.content)
+            except requests.RequestException as e:
+                logger.error(f"Error downloading image {image_url}: {e}")
+
+        # 画像フォルダをパスワード付きZIP
         try:
-            subprocess.run(cmd, check=True)
-            logger.info("[OK] rclone copy => cryptLocal:")
+            original_cwd = os.getcwd()
+            os.chdir(folder)
+            subprocess.run(['zip', '-e', '--password=osanpo', 'images.zip', '-r', 'images'], check=True)
+            logger.info(f"Images compressed and saved to {os.path.join(folder, 'images.zip')}")
         except subprocess.CalledProcessError as e:
-            logger.error(f"[ERR] rclone copy failed: {e}")
-            return None
-
-        after = set(os.listdir(encrypted_dir))
-        diff = after - before
-        if not diff:
-            logger.error("[ERR] no new directory in ./encrypted after copy")
-            return None
-        if len(diff) > 1:
-            logger.warning(f"[WARN] multiple new dirs => {diff}")
-        enc_name = diff.pop()
-        enc_path = os.path.join(encrypted_dir, enc_name)
-        if not os.path.isdir(enc_path):
-            logger.error(f"[ERR] {enc_path} is not a directory.")
-            return None
-
-        # HF upload folder
+            logger.error(f"Error creating zip file: {e}")
+        finally:
+            os.chdir(original_cwd)
+
+        if os.path.exists(images_folder):
+            shutil.rmtree(images_folder)
+
+    def save_html_content(self, url: str, folder: str):
+        """指定されたURLからHTMLコンテンツを取得し、保存する。"""
         try:
-            self.upload_folder(enc_path, path_in_repo=enc_name)
+            response = requests.get(url)
+            response.raise_for_status()
+            html_path = os.path.join(folder, f"{folder}.html")
+            with open(html_path, 'w', encoding='utf-8') as file:
+                file.write(response.text)
         except Exception as e:
-            logger.error(f"[ERR] encrypt_and_upload_folder => upload_folder: {e}")
+            logger.error(f"Error saving HTML content for URL {url}: {e}")
+
+    @staticmethod
+    def save_model_info(model_info: dict, folder: str):
+        """モデル情報(json)の保存"""
+        with open(os.path.join(folder, "model_info.json"), "w") as file:
+            json.dump(model_info, file, indent=2)
+
+    @staticmethod
+    def increment_repo_name(repo_id: str) -> str:
+        """リポジトリ名の末尾の数字をインクリメントする。"""
+        match = re.search(r'(\d+)$', repo_id)
+        if match:
+            number = int(match.group(1)) + 1
+            new_repo_id = re.sub(r'\d+$', str(number), repo_id)
+        else:
+            new_repo_id = f"{repo_id}1"
+        return new_repo_id
 
-        # cleanup
-        shutil.rmtree(local_folder, ignore_errors=True)
-        shutil.rmtree(enc_path, ignore_errors=True)
-        logger.info(f"[CLEANUP] removed {local_folder} & {enc_path}")
+    # =============================
+    # ここを rclone 暗号化＆アップロードに書き換え
+    # =============================
+    def upload_file(self, file_path: str, repo_id: Optional[str] = None, path_in_repo: Optional[str] = None):
+        """
+        1) rcloneで file_path を暗号化
+        2) 暗号化されたファイル(群)を Hugging Face にアップロード
+        """
+        if repo_id is None:
+            repo_id = self.repo_ids['current']
+        if path_in_repo is None:
+            path_in_repo = os.path.basename(file_path)
 
-        # ★ 成功したら enc_name を返す
-        return enc_name
+        # 1) rclone copy (ファイル暗号化)
+        self.encrypt_with_rclone(file_path, is_file=True)
 
-    def download_and_process_versions(self, model_versions: list, folder: str):
-        if not model_versions:
-            return
+        # 2) 暗号ファイルをアップロード
+        self.upload_encrypted_files(repo_id=repo_id, path_in_repo=path_in_repo)
 
-        latest = model_versions[0]
-        for f_info in latest.get("files", []):
-            url = f_info["downloadUrl"]
-            fname = f_info["name"]
-            self.download_file(url, folder, fname)
+        # 3) 暗号ディレクトリの掃除
+        if os.path.isdir(self.config.ENCRYPTED_DIR):
+            shutil.rmtree(self.config.ENCRYPTED_DIR, ignore_errors=True)
 
-        if len(model_versions) > 1:
-            ov_folder = os.path.join(folder, "old_versions")
-            os.makedirs(ov_folder, exist_ok=True)
-            for v in model_versions[1:]:
-                for f_info in v.get("files", []):
-                    url = f_info["downloadUrl"]
-                    fname = f_info["name"]
-                    self.download_file(url, ov_folder, fname)
+    def upload_folder(self, folder_path: str, path_in_repo: Optional[str] = None):
+        """
+        1) rcloneで folder_path を暗号化
+        2) 暗号化されたフォルダを Hugging Face にアップロード
+        """
+        if path_in_repo is None:
+            path_in_repo = os.path.basename(folder_path)
 
-    def get_model_info(self, model_id: str) -> dict:
+        # 1) rclone copy (フォルダ暗号化)
+        self.encrypt_with_rclone(folder_path, is_file=False)
+
+        # 2) 暗号フォルダをアップロード
+        self.upload_encrypted_files(repo_id=self.repo_ids['current'], path_in_repo=path_in_repo)
+
+        # 3) 掃除
+        if os.path.isdir(self.config.ENCRYPTED_DIR):
+            shutil.rmtree(self.config.ENCRYPTED_DIR, ignore_errors=True)
+
+    def read_model_list(self):
+        """モデルリストを読み込む。"""
+        model_list = {}
         try:
-            url = f"{self.config.URLS['modelId']}{model_id}"
-            resp = requests.get(url, headers=self.config.HEADERS)
-            resp.raise_for_status()
-            return resp.json()
+            with open(self.config.LIST_FILE, "r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if line:
+                        parts = line.split(": ", 1)
+                        if len(parts) == 2:
+                            modelpage_name, model_hf_url = parts
+                            model_list[model_hf_url] = modelpage_name
+            return model_list
         except Exception as e:
-            logger.error(f"[ERR] get_model_info({model_id}): {e}")
+            logger.error(f"Failed to read model list: {e}")
             return {}
 
-    def download_images(self, model_versions: list, folder: str):
-        """
-        各model_versionsから画像URLを集めて、
-        folder/images 下にダウンロードするメソッド。
-        """
-        images_folder = os.path.join(folder, "images")
-        os.makedirs(images_folder, exist_ok=True)
-
-        for version in model_versions:
-            for img_info in version.get("images", []):
-                img_url = img_info["url"]
-                filename = os.path.basename(img_url)
-                self.download_file(img_url, images_folder, filename)
-
-    def process_model(self, model_id: str):
-        info = self.get_model_info(model_id)
-        if not info or "modelVersions" not in info:
-            logger.error(f"[ERR] No modelVersions for {model_id}")
-            return
-        
-        versions = info["modelVersions"]
-        base_dir = "local_models"
-        os.makedirs(base_dir, exist_ok=True)
-
-        # モデル名
-        model_name = info.get("name", f"ID_{model_id}")
-        safe_name = re.sub(r'[\\/*?:"<>|]', '_', model_name)  # OSで使えない文字を _
-        folder_path = os.path.join(base_dir, safe_name)
-        if os.path.exists(folder_path):
-            shutil.rmtree(folder_path)
-        os.makedirs(folder_path, exist_ok=True)
-        logger.info(f"[OK] Created local folder => {folder_path}")
-
-        # ダウンロード
-        self.download_and_process_versions(versions, folder_path)
-        self.download_images(versions, folder_path)
-
-        # === 暗号化＆アップロード ===
-        logger.info(f"[DEBUG] encrypt_and_upload_folder => {folder_path}")
-        enc_subfolder = self.encrypt_and_upload_folder(folder_path)
-        if enc_subfolder is None:
-            # 失敗
-            enc_subfolder = "[ENCRYPT_FAILED]"
-        else:
-            logger.info(f"[OK] Encrypted & uploaded => {enc_subfolder}")
+    def get_repo_info(self, repo_id):
+        """リポジトリの情報を取得する。"""
+        try:
+            repo_info = self.api.repo_info(repo_id=repo_id, files_metadata=True)
+            file_paths = [sibling.rfilename for sibling in repo_info.siblings]
+            return file_paths
+        except Exception as e:
+            logger.error(f"Failed to get repo info for {repo_id}: {e}")
+            return []
 
-        # ★ model_list.log に追記
-        #   enc_subfolder が "[ENCRYPT_FAILED]" でなければ成功したフォルダ名
-        hf_url = f"https://huggingface.co/{self.repo_ids['current']}/tree/main/{enc_subfolder}"
-        model_list_line = f"{model_name} (ID:{model_id}): {hf_url}\n"
+    def process_model(self, model_url: str):
+        """指定されたモデルURLを処理する関数。"""
         try:
+            model_id = model_url.rstrip("/").split("/")[-1]
+            model_info = self.get_model_info(model_id)
+
+            latest_version = model_info.get("modelVersions", [])[0]
+            model_file = next(
+                (file for file in latest_version["files"] if file.get('type') == 'Model'),
+                None
+            )
+            if model_file:
+                latest_filename = model_file['name']
+                folder = os.path.splitext(latest_filename)[0]
+            else:
+                first_file = latest_version["files"][0]
+                latest_filename = first_file['name']
+                folder = os.path.splitext(latest_filename)[0]
+                logger.warning(f"No 'Model' type file found for model ID {model_id}. Using first file's name.")
+
+            os.makedirs(folder, exist_ok=True)
+            model_hf_url = f"https://huggingface.co/{self.repo_ids['current']}/tree/main/{folder}"
+            model_list = self.read_model_list()
+
+            if model_hf_url in model_list:
+                repo_id = self.repo_ids['current']
+                repo_files = self.get_repo_info(repo_id)
+                old_versions_files = [f for f in repo_files if f.startswith(f"{folder}/old_versions/")]
+                existing_old_version_files = [os.path.basename(f) for f in old_versions_files]
+            else:
+                existing_old_version_files = []
+
+            self.download_model(model_info["modelVersions"], folder, existing_old_version_files)
+            self.download_images(model_info["modelVersions"], folder)
+            self.save_html_content(model_url, folder)
+            self.save_model_info(model_info, folder)
+
+            # ====== rclone でフォルダ暗号化 → HFへアップロード ======
+            self.upload_folder(folder)
+
+            # モデルリスト更新
+            modelpage_name = model_info.get("name", "Unnamed Model")
+            model_hf_url = f"https://huggingface.co/{self.repo_ids['current']}/tree/main/{folder}"
             with open(self.config.LIST_FILE, "a", encoding="utf-8") as f:
-                f.write(model_list_line)
-            logger.info(f"[OK] Wrote to model_list.log => {model_list_line.strip()}")
-        except Exception as e:
-            logger.error(f"[ERR] writing model_list.log => {e}")
+                f.write(f"{modelpage_name}: {model_hf_url}\n")
 
-        # ★ model_list.log をアップロード
-        self.upload_file(self.config.LIST_FILE, self.repo_ids["model_list"], self.config.LIST_FILE)
+            # ローカルフォルダを削除
+            if os.path.exists(folder):
+                shutil.rmtree(folder)
+
+        except Exception as e:
+            logger.error(f"Unexpected error processing model ({model_url}): {e}")
 
     async def crawl(self):
+        """モデルを定期的にチェックし、更新を行う。"""
         while True:
             try:
                 login(token=self.config.HUGGINGFACE_API_KEY, add_to_git_credential=True)
 
-                model_list_path = hf_hub_download(self.repo_ids["model_list"], self.config.LIST_FILE)
+                # model_list.logのダウンロード
+                model_list_path = hf_hub_download(repo_id=self.repo_ids['model_list'], filename=self.config.LIST_FILE)
                 shutil.copyfile(model_list_path, f"./{self.config.LIST_FILE}")
 
-                log_path = hf_hub_download(self.repo_ids["log"], self.config.LOG_FILE)
-                shutil.copyfile(log_path, f"./{self.config.LOG_FILE}")
-
-                with open(self.config.LOG_FILE, "r", encoding="utf-8") as f:
-                    lines = f.read().splitlines()
-                    old_models = json.loads(lines[0]) if len(lines)>0 else []
-                    self.repo_ids["current"] = lines[1] if len(lines)>1 else ""
-
-                # get newest
-                r = requests.get(self.config.URLS["latest"], headers=self.config.HEADERS)
-                r.raise_for_status()
-                items = r.json().get("items", [])
-                new_ids = [it["id"] for it in items if "id" in it]
-
-                diff_ids = list(set(new_ids) - set(old_models))
-                if diff_ids:
-                    mid = diff_ids[0]
-                    for attempt in range(1,6):
+                # ログファイルのダウンロード
+                local_file_path = hf_hub_download(repo_id=self.repo_ids["log"], filename=self.config.LOG_FILE)
+                shutil.copyfile(local_file_path, f"./{self.config.LOG_FILE}")
+
+                # ログ読み込み
+                with open(self.config.LOG_FILE, "r", encoding="utf-8") as file:
+                    lines = file.read().splitlines()
+                    old_models = json.loads(lines[0]) if len(lines) > 0 else []
+                    self.repo_ids["current"] = lines[1] if len(lines) > 1 else ""
+
+                # 新着モデルの取得
+                response = requests.get(self.config.URLS["latest"], headers=self.config.HEADERS)
+                response.raise_for_status()
+                latest_models = response.json().get("items", [])
+                latest_model_ids = [item.get("id") for item in latest_models if "id" in item]
+
+                # 増分の確認
+                new_models = list(set(latest_model_ids) - set(old_models))
+
+                if new_models:
+                    logger.info(f"New models found: {new_models}")
+                    model_id = new_models[0]
+                    for attempt in range(1, 6):
                         try:
-                            self.process_model(str(mid))
+                            self.process_model(f"{self.config.URLS['modelId']}{model_id}")
                             break
                         except Exception as e:
-                            logger.error(f"[ERR] process_model {mid} (attempt {attempt}): {e}")
-                            if attempt==5:
-                                logger.error(f"[SKIP] model {mid} after 5 fails")
+                            logger.error(f"Failed to process model ID {model_id} (Attempt {attempt}/5): {e}")
+                            if attempt == 5:
+                                logger.error(f"Skipping model ID {model_id} after 5 failed attempts.")
                             else:
                                 await asyncio.sleep(2)
-
-                    old_models.append(mid)
-                    with open(self.config.LOG_FILE,'w',encoding='utf-8') as f:
-                        f.write(json.dumps(old_models)+"\n")
-                        f.write(self.repo_ids["current"]+"\n")
-
-                    # アップロードログ
-                    self.upload_file(self.config.LOG_FILE, self.repo_ids["log"], self.config.LOG_FILE)
                 else:
-                    with open(self.config.LOG_FILE,'w',encoding='utf-8') as f:
-                        f.write(json.dumps(new_ids)+"\n")
-                        f.write(self.repo_ids["current"]+"\n")
-                    self.upload_file(self.config.LOG_FILE, self.repo_ids["log"], self.config.LOG_FILE)
-                    logger.info("[INFO] no new models => sleep(60)")
+                    # ログファイルを最新のモデルIDで上書き
+                    with open(self.config.LOG_FILE, "w", encoding="utf-8") as f:
+                        f.write(json.dumps(latest_model_ids) + "\n")
+                        f.write(f"{self.repo_ids['current']}\n")
+                    logger.info(f"Updated log file: {self.config.LOG_FILE}")
+
+                    # ログファイルをリポジトリにアップロード
+                    self.upload_file(
+                        file_path=self.config.LOG_FILE,
+                        repo_id=self.repo_ids["log"],
+                        path_in_repo=self.config.LOG_FILE
+                    )
+                    logger.info("Uploaded log file to repository.")
+
+                    logger.info("No new models found.")
                     await asyncio.sleep(60)
                     continue
+
+                # 古いモデルリストに追加
+                old_models.append(model_id)
+
+                # ログファイルの更新
+                with open(self.config.LOG_FILE, "w", encoding="utf-8") as f:
+                    f.write(json.dumps(old_models) + "\n")
+                    f.write(f"{self.repo_ids['current']}\n")
+                logger.info(f"Updated log file with new model ID: {model_id}")
+
+                # ログとモデルリストのアップロード (rcloneは使わず直接uploadでもOKならそのまま)
+                self.upload_file(
+                    file_path=self.config.LOG_FILE,
+                    repo_id=self.repo_ids["log"],
+                    path_in_repo=self.config.LOG_FILE
+                )
+                self.upload_file(
+                    file_path=self.config.LIST_FILE,
+                    repo_id=self.repo_ids["model_list"],
+                    path_in_repo=self.config.LIST_FILE
+                )
+
             except Exception as e:
-                logger.error(f"[ERR] crawl => {e}")
+                logger.error(f"Error during crawling: {e}")
                 await asyncio.sleep(300)
 
 
-# FastAPI
+# モジュールレベルでFastAPIのアプリケーションを公開
 config = Config()
 crawler = CivitAICrawler(config)
 app = crawler.app