HuggingFace0920 commited on
Commit
4fcdb0e
·
verified ·
1 Parent(s): 2cf26e4

Update sync_data.sh

Browse files
Files changed (1) hide show
  1. sync_data.sh +30 -38
sync_data.sh CHANGED
@@ -3,20 +3,14 @@
3
  # 检查环境变量
4
  if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
5
  echo "未检测到 HF_TOKEN 或 DATASET_ID,备份功能不可用"
6
- # 修改为返回 0,避免容器启动失败
7
- exit 0
8
  fi
9
 
10
- # 激活虚拟环境(如果需要)
11
- if [ -d "$HOME/venv" ]; then
12
- echo "激活虚拟环境..."
13
- . $HOME/venv/bin/activate
14
- else
15
- echo "未找到虚拟环境,使用系统 Python..."
16
- fi
17
 
18
  # 生成同步脚本
19
- cat > /app/hf_sync.py << 'EOL'
20
  # HuggingFace 同步脚本
21
  from huggingface_hub import HfApi
22
  import sys
@@ -53,6 +47,7 @@ def upload_backup(file_path, file_name, token, repo_id):
53
  except Exception as e:
54
  print(f"上传文件出错: {str(e)}")
55
 
 
56
  # 下载最新备份
57
  def download_latest_backup(token, repo_id, extract_path):
58
  try:
@@ -101,56 +96,53 @@ if __name__ == "__main__":
101
  elif action == "super_squash":
102
  super_squash_history(token, repo_id)
103
  EOL
104
-
105
  # 首次启动时从 HuggingFace 下载最新备份(解压到应用目录)
106
  echo "正在从 HuggingFace 下载最新备份..."
107
- python3 /app/hf_sync.py download "${HF_TOKEN}" "${DATASET_ID}" "/app/data"
108
 
109
  # 同步函数
110
  sync_data() {
111
  while true; do
112
  echo "同步进程启动于 $(date)"
113
 
114
- # 确保数据目录存在
115
- DATA_DIR="/app/data"
116
- DB_FILE="${DATA_DIR}/webui.db"
117
-
118
- if [ -f "${DB_FILE}" ]; then
119
  # 创建备份
120
  timestamp=$(date +%Y%m%d_%H%M%S)
121
  backup_file="backup_${timestamp}.tar.gz"
122
 
123
- # 压缩文件
124
- tar -czf "/tmp/${backup_file}" -C "${DATA_DIR}" "webui.db"
125
 
126
  # 上传到 HuggingFace
127
  echo "正在上传备份到 HuggingFace..."
128
- python3 /app/hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
129
 
130
  # 合并历史提交
131
- SQUASH_FLAG_FILE="/tmp/last_squash_time"
132
- NOW=$(date +%s)
133
- SEVEN_DAYS=$((7*24*60*60))
134
- if [ ! -f "$SQUASH_FLAG_FILE" ]; then
 
 
 
 
 
 
 
135
  echo $NOW > "$SQUASH_FLAG_FILE"
136
- echo "首次合并历史提交..."
137
- python3 /app/hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}"
138
  else
139
- LAST=$(cat "$SQUASH_FLAG_FILE")
140
- DIFF=$((NOW - LAST))
141
- if [ $DIFF -ge $SEVEN_DAYS ]; then
142
- echo $NOW > "$SQUASH_FLAG_FILE"
143
- echo "距离上次合并已超过7天,正在合并历史提交..."
144
- python3 /app/hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}"
145
- else
146
- echo "距离上次合并未满7天,本次跳过合并历史提交。"
147
- fi
148
  fi
 
149
 
150
  # 清理临时文件
151
  rm -f "/tmp/${backup_file}"
152
  else
153
- echo "数据库文件 ${DB_FILE} 不存在,等待中..."
154
  fi
155
 
156
  # 同步间隔
@@ -160,5 +152,5 @@ sync_data() {
160
  done
161
  }
162
 
163
- # 启动同步进程(在后台运行)
164
- sync_data &
 
3
  # 检查环境变量
4
  if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
5
  echo "未检测到 HF_TOKEN 或 DATASET_ID,备份功能不可用"
6
+ exit 1
 
7
  fi
8
 
9
+ # 激活虚拟环境
10
+ . $HOME/venv/bin/activate
 
 
 
 
 
11
 
12
  # 生成同步脚本
13
+ cat > hf_sync.py << 'EOL'
14
  # HuggingFace 同步脚本
15
  from huggingface_hub import HfApi
16
  import sys
 
47
  except Exception as e:
48
  print(f"上传文件出错: {str(e)}")
49
 
50
+
51
  # 下载最新备份
52
  def download_latest_backup(token, repo_id, extract_path):
53
  try:
 
96
  elif action == "super_squash":
97
  super_squash_history(token, repo_id)
98
  EOL
 
99
  # 首次启动时从 HuggingFace 下载最新备份(解压到应用目录)
100
  echo "正在从 HuggingFace 下载最新备份..."
101
+ python hf_sync.py download "${HF_TOKEN}" "${DATASET_ID}" "$HOME/app"
102
 
103
  # 同步函数
104
  sync_data() {
105
  while true; do
106
  echo "同步进程启动于 $(date)"
107
 
108
+ # 确保数据目录存在(请根据实际路径修改)
109
+ STORAGE_PATH="$HOME/app/data"
110
+ if [ -d "${STORAGE_PATH}" ]; then
 
 
111
  # 创建备份
112
  timestamp=$(date +%Y%m%d_%H%M%S)
113
  backup_file="backup_${timestamp}.tar.gz"
114
 
115
+ # 压缩目录(使用-C避免包含父路径)
116
+ tar -czf "/tmp/${backup_file}" -C "$(dirname "${STORAGE_PATH}")" "$(basename "${STORAGE_PATH}")"
117
 
118
  # 上传到 HuggingFace
119
  echo "正在上传备份到 HuggingFace..."
120
+ python hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
121
 
122
  # 合并历史提交
123
+ SQUASH_FLAG_FILE="/tmp/last_squash_time"
124
+ NOW=$(date +%s)
125
+ SEVEN_DAYS=$((7*24*60*60))
126
+ if [ ! -f "$SQUASH_FLAG_FILE" ]; then
127
+ echo $NOW > "$SQUASH_FLAG_FILE"
128
+ echo "首次合并历史提交..."
129
+ python hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}"
130
+ else
131
+ LAST=$(cat "$SQUASH_FLAG_FILE")
132
+ DIFF=$((NOW - LAST))
133
+ if [ $DIFF -ge $SEVEN_DAYS ]; then
134
  echo $NOW > "$SQUASH_FLAG_FILE"
135
+ echo "距离上次合并已超过7天,正在合并历史提交..."
136
+ python hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}"
137
  else
138
+ echo "距离上次合并未满7天,本次跳过合并历史提交。"
 
 
 
 
 
 
 
 
139
  fi
140
+ fi
141
 
142
  # 清理临时文件
143
  rm -f "/tmp/${backup_file}"
144
  else
145
+ echo "存储目录 ${STORAGE_PATH} 不存在,等待中..."
146
  fi
147
 
148
  # 同步间隔
 
152
  done
153
  }
154
 
155
+ # 启动同步进程
156
+ sync_data &