HuggingFace0920 commited on
Commit
2cf26e4
·
verified ·
1 Parent(s): b60299a

Update sync_data.sh

Browse files
Files changed (1) hide show
  1. sync_data.sh +38 -30
sync_data.sh CHANGED
@@ -3,14 +3,20 @@
3
  # 检查环境变量
4
  if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
5
  echo "未检测到 HF_TOKEN 或 DATASET_ID,备份功能不可用"
6
- exit 1
 
7
  fi
8
 
9
- # 激活虚拟环境
10
- . $HOME/venv/bin/activate
 
 
 
 
 
11
 
12
  # 生成同步脚本
13
- cat > hf_sync.py << 'EOL'
14
  # HuggingFace 同步脚本
15
  from huggingface_hub import HfApi
16
  import sys
@@ -47,7 +53,6 @@ def upload_backup(file_path, file_name, token, repo_id):
47
  except Exception as e:
48
  print(f"上传文件出错: {str(e)}")
49
 
50
-
51
  # 下载最新备份
52
  def download_latest_backup(token, repo_id, extract_path):
53
  try:
@@ -96,53 +101,56 @@ if __name__ == "__main__":
96
  elif action == "super_squash":
97
  super_squash_history(token, repo_id)
98
  EOL
 
99
  # 首次启动时从 HuggingFace 下载最新备份(解压到应用目录)
100
  echo "正在从 HuggingFace 下载最新备份..."
101
- python hf_sync.py download "${HF_TOKEN}" "${DATASET_ID}" "./data"
102
 
103
  # 同步函数
104
  sync_data() {
105
  while true; do
106
  echo "同步进程启动于 $(date)"
107
 
108
- # 确保数据目录存在(请根据实际路径修改)
109
- STORAGE_PATH="./data/webui.db"
110
- if [ -d "${STORAGE_PATH}" ]; then
 
 
111
  # 创建备份
112
  timestamp=$(date +%Y%m%d_%H%M%S)
113
  backup_file="backup_${timestamp}.tar.gz"
114
 
115
- # 压缩目录(使用-C避免包含父路径)
116
- tar -czf "/tmp/${backup_file}" -C "$(dirname "${STORAGE_PATH}")" "$(basename "${STORAGE_PATH}")"
117
 
118
  # 上传到 HuggingFace
119
  echo "正在上传备份到 HuggingFace..."
120
- python hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
121
 
122
  # 合并历史提交
123
- SQUASH_FLAG_FILE="/tmp/last_squash_time"
124
- NOW=$(date +%s)
125
- SEVEN_DAYS=$((7*24*60*60))
126
- if [ ! -f "$SQUASH_FLAG_FILE" ]; then
127
- echo $NOW > "$SQUASH_FLAG_FILE"
128
- echo "首次合并历史提交..."
129
- python hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}"
130
- else
131
- LAST=$(cat "$SQUASH_FLAG_FILE")
132
- DIFF=$((NOW - LAST))
133
- if [ $DIFF -ge $SEVEN_DAYS ]; then
134
  echo $NOW > "$SQUASH_FLAG_FILE"
135
- echo "距离上次合并已超过7天,正在合并历史提交..."
136
- python hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}"
137
  else
138
- echo "距离上次合并未满7天,本次跳过合并历史提交。"
 
 
 
 
 
 
 
 
139
  fi
140
- fi
141
 
142
  # 清理临时文件
143
  rm -f "/tmp/${backup_file}"
144
  else
145
- echo "存储目录 ${STORAGE_PATH} 不存在,等待中..."
146
  fi
147
 
148
  # 同步间隔
@@ -152,5 +160,5 @@ sync_data() {
152
  done
153
  }
154
 
155
- # 启动同步进程
156
- sync_data &
 
3
  # 检查环境变量
4
  if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
5
  echo "未检测到 HF_TOKEN 或 DATASET_ID,备份功能不可用"
6
+ # 修改为返回 0,避免容器启动失败
7
+ exit 0
8
  fi
9
 
10
+ # 激活虚拟环境(如果需要)
11
+ if [ -d "$HOME/venv" ]; then
12
+ echo "激活虚拟环境..."
13
+ . $HOME/venv/bin/activate
14
+ else
15
+ echo "未找到虚拟环境,使用系统 Python..."
16
+ fi
17
 
18
  # 生成同步脚本
19
+ cat > /app/hf_sync.py << 'EOL'
20
  # HuggingFace 同步脚本
21
  from huggingface_hub import HfApi
22
  import sys
 
53
  except Exception as e:
54
  print(f"上传文件出错: {str(e)}")
55
 
 
56
  # 下载最新备份
57
  def download_latest_backup(token, repo_id, extract_path):
58
  try:
 
101
  elif action == "super_squash":
102
  super_squash_history(token, repo_id)
103
  EOL
104
+
105
  # 首次启动时从 HuggingFace 下载最新备份(解压到应用目录)
106
  echo "正在从 HuggingFace 下载最新备份..."
107
+ python3 /app/hf_sync.py download "${HF_TOKEN}" "${DATASET_ID}" "/app/data"
108
 
109
  # 同步函数
110
  sync_data() {
111
  while true; do
112
  echo "同步进程启动于 $(date)"
113
 
114
+ # 确保数据目录存在
115
+ DATA_DIR="/app/data"
116
+ DB_FILE="${DATA_DIR}/webui.db"
117
+
118
+ if [ -f "${DB_FILE}" ]; then
119
  # 创建备份
120
  timestamp=$(date +%Y%m%d_%H%M%S)
121
  backup_file="backup_${timestamp}.tar.gz"
122
 
123
+ # 压缩文件
124
+ tar -czf "/tmp/${backup_file}" -C "${DATA_DIR}" "webui.db"
125
 
126
  # 上传到 HuggingFace
127
  echo "正在上传备份到 HuggingFace..."
128
+ python3 /app/hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
129
 
130
  # 合并历史提交
131
+ SQUASH_FLAG_FILE="/tmp/last_squash_time"
132
+ NOW=$(date +%s)
133
+ SEVEN_DAYS=$((7*24*60*60))
134
+ if [ ! -f "$SQUASH_FLAG_FILE" ]; then
 
 
 
 
 
 
 
135
  echo $NOW > "$SQUASH_FLAG_FILE"
136
+ echo "首次合并历史提交..."
137
+ python3 /app/hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}"
138
  else
139
+ LAST=$(cat "$SQUASH_FLAG_FILE")
140
+ DIFF=$((NOW - LAST))
141
+ if [ $DIFF -ge $SEVEN_DAYS ]; then
142
+ echo $NOW > "$SQUASH_FLAG_FILE"
143
+ echo "距离上次合并已超过7天,正在合并历史提交..."
144
+ python3 /app/hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}"
145
+ else
146
+ echo "距离上次合并未满7天,本次跳过合并历史提交。"
147
+ fi
148
  fi
 
149
 
150
  # 清理临时文件
151
  rm -f "/tmp/${backup_file}"
152
  else
153
+ echo "数据库文件 ${DB_FILE} 不存在,等待中..."
154
  fi
155
 
156
  # 同步间隔
 
160
  done
161
  }
162
 
163
+ # 启动同步进程(在后台运行)
164
+ sync_data &