Upload main.py

main.py

@@ -128,11 +128,8 @@ class CivitAICrawler:
                 f"[ERROR] {self.config.ENCRYPTED_DIR} not found. Check your rclone config."
             )
 
+    # 例: upload_encrypted_files の中の再試行処理
     def upload_encrypted_files(self, repo_id: str, base_path_in_repo: str = ""):
-        """
-        self.config.ENCRYPTED_DIR 以下に生成された暗号化後のファイル・フォルダ構造を再帰的に
-        Hugging Face にアップロードする。フォルダ名・ファイル名は rclone により暗号化済み。
-        """
         max_retries = 5
         for root, dirs, files in os.walk(self.config.ENCRYPTED_DIR):
             for fn in files:
@@ -153,9 +150,37 @@ class CivitAICrawler:
                         )
                         logger.info(f"[OK] Uploaded => {repo_id}/{upload_path_in_repo}")
                         break
+
                     except Exception as e:
                         attempt += 1
                         error_message = str(e)
+
+                        # ================================
+                        # 429によるrate-limit検出追加
+                        # ================================
+                        # "You have been rate-limited; you can retry this action in 31 minutes."
+                        # のようなメッセージから時間を抽出し、その時間+1分だけ待機後、再試行
+                        if "rate-limited" in error_message and "minutes" in error_message:
+                            import re
+                            match = re.search(r"in (\d+) minutes?", error_message)
+                            if match:
+                                minutes = int(match.group(1))
+                                # +1分して待機
+                                minutes += 1
+                                logger.warning(f"Rate-limited. Waiting {minutes} minutes before retry...")
+                                time.sleep(minutes * 60)
+                                attempt -= 1  # 同じ attempt カウントで再試行
+                                continue
+
+                        # ================================
+                        # すでにある1時間待機処理
+                        # ================================
+                        if "you can retry this action in about 1 hour" in error_message:
+                            logger.warning("Encountered 'retry in 1 hour' error. Waiting 1 hour before retrying...")
+                            time.sleep(3600)
+                            attempt -= 1  # 再試行回数を増やさずにループを続ける
+                            continue
+
                         if "over the limit of 100000 files" in error_message:
                             logger.warning("Repository file limit exceeded. Creating a new repository...")
                             self.repo_ids['current'] = self.increment_repo_name(self.repo_ids['current'])
@@ -163,20 +188,17 @@ class CivitAICrawler:
                             attempt = 0
                             repo_id = self.repo_ids['current']
                             continue
-                        elif "you can retry this action in about 1 hour" in error_message:
-                            logger.warning("Encountered 'retry in 1 hour' error. Waiting 1 hour before retrying...")
-                            time.sleep(3600)
-                            attempt -= 1
+
+                        # 上記以外のエラーの場合
+                        if attempt < max_retries:
+                            logger.warning(
+                                f"Failed to upload {encrypted_file_path}, retry {attempt}/{max_retries}..."
+                            )
                         else:
-                            if attempt < max_retries:
-                                logger.warning(
-                                    f"Failed to upload {encrypted_file_path}, retry {attempt}/{max_retries}..."
-                                )
-                            else:
-                                logger.error(
-                                    f"Failed to upload after {max_retries} attempts: {encrypted_file_path}"
-                                )
-                                raise
+                            logger.error(
+                                f"Failed to upload after {max_retries} attempts: {encrypted_file_path}"
+                            )
+                            raise
 
     @staticmethod
     def get_filename_from_cd(content_disposition: Optional[str], default_name: str) -> str: