hxger commited on
Commit
437ef8f
·
verified ·
1 Parent(s): 79b3375

Upload 4 files

Browse files
Files changed (4) hide show
  1. .gitignore +25 -0
  2. Dockerfile +67 -0
  3. launch.sh +55 -0
  4. sync_data.sh +140 -0
.gitignore ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 临时文件
2
+ *.log
3
+ *.tmp
4
+ .DS_Store
5
+
6
+ # 运行时数据
7
+ /data/
8
+ /config/
9
+ /uploads/
10
+ /cache/
11
+ /backups/
12
+ /thumbnails/
13
+
14
+ # Node.js
15
+ node_modules/
16
+ npm-debug.log
17
+
18
+ # 编辑器配置文件
19
+ .idea/
20
+ .vscode/
21
+ *.swp
22
+ *.swo
23
+
24
+ # 本地环境变量
25
+ .env
Dockerfile ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM node:19.1.0-alpine3.16
2
+
3
+ # 设置应用目录
4
+ ARG APP_HOME=/home/node/app
5
+
6
+ # 安装系统依赖
7
+ RUN apk add gcompat tini git python3 py3-pip
8
+
9
+ # 安装HuggingFace Hub
10
+ RUN pip3 install --no-cache-dir huggingface_hub
11
+
12
+ # 确保正确处理内核信号
13
+ ENTRYPOINT [ "tini", "--" ]
14
+
15
+ # 创建应用目录
16
+ WORKDIR ${APP_HOME}
17
+
18
+ # 设置NODE_ENV为production
19
+ ENV NODE_ENV=production
20
+
21
+ # 设置登录凭证环境变量
22
+ ENV USERNAME="admin"
23
+ ENV PASSWORD="password"
24
+
25
+ # 克隆官方SillyTavern仓库(最新版本)
26
+ RUN git clone https://github.com/SillyTavern/SillyTavern.git .
27
+
28
+ # 安装依赖
29
+ RUN echo "*** 安装npm包 ***" && \
30
+ npm install && npm cache clean --force
31
+
32
+ # 添加启动脚本和数据同步脚本
33
+ COPY launch.sh sync_data.sh ./
34
+ RUN chmod +x launch.sh sync_data.sh
35
+
36
+ # 安装生产依赖
37
+ RUN echo "*** 安装生产npm包 ***" && \
38
+ npm i --no-audit --no-fund --loglevel=error --no-progress --omit=dev && npm cache clean --force
39
+
40
+ # 创建配置目录
41
+ RUN mkdir -p "config" || true && \
42
+ rm -f "config.yaml" || true && \
43
+ ln -s "./config/config.yaml" "config.yaml" || true
44
+
45
+ # 清理不必要的文件
46
+ RUN echo "*** 清理 ***" && \
47
+ mv "./docker/docker-entrypoint.sh" "./" && \
48
+ rm -rf "./docker" && \
49
+ echo "*** 使docker-entrypoint.sh可执行 ***" && \
50
+ chmod +x "./docker-entrypoint.sh" && \
51
+ echo "*** 转换行尾为Unix格式 ***" && \
52
+ dos2unix "./docker-entrypoint.sh" || true
53
+
54
+ # 修改入口脚本,添加自定义启动脚本
55
+ RUN sed -i 's/# Start the server/.\/launch.sh/g' docker-entrypoint.sh
56
+
57
+ # 创建临时备份目录
58
+ RUN mkdir -p /tmp/sillytavern_backup
59
+
60
+ # 设置权限
61
+ RUN chmod -R 777 ${APP_HOME}
62
+
63
+ # 暴露端口
64
+ EXPOSE 8000
65
+
66
+ # 启动命令
67
+ CMD [ "./docker-entrypoint.sh" ]
launch.sh ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/sh
2
+
3
+ BASE=/home/node/app
4
+ USERNAME=$(printenv USERNAME)
5
+ PASSWORD=$(printenv PASSWORD)
6
+ HF_TOKEN=$(printenv HF_TOKEN)
7
+ DATASET_ID=$(printenv DATASET_ID)
8
+ SYNC_INTERVAL=$(printenv SYNC_INTERVAL)
9
+
10
+ # 如果没有设置用户名和密码,使用默认值
11
+ if [[ -z "${USERNAME}" ]]; then
12
+ USERNAME="admin"
13
+ fi
14
+
15
+ if [[ -z "${PASSWORD}" ]]; then
16
+ PASSWORD="password"
17
+ fi
18
+
19
+ echo
20
+ echo "用户名: ${USERNAME}"
21
+ echo "密码: ${PASSWORD}"
22
+ echo
23
+
24
+ # 确保配置目录存在
25
+ mkdir -p "${BASE}/config"
26
+
27
+ # 如果配置文件不存在,从默认目录复制
28
+ if [ ! -e "${BASE}/config/config.yaml" ]; then
29
+ echo "配置文件不存在,从默认目录复制: config.yaml"
30
+ cp -r "${BASE}/default/config.yaml" "${BASE}/config/config.yaml"
31
+ fi
32
+
33
+ # 修改配置文件中的用户名和密码
34
+ sed -i "s/username: .*/username: \"${USERNAME}\"/" ${BASE}/config/config.yaml
35
+ sed -i "s/password: .*/password: \"${PASSWORD}\"/" ${BASE}/config/config.yaml
36
+
37
+ # 启用基本认证模式,禁用白名单模式
38
+ sed -i "s/whitelistMode: true/whitelistMode: false/" ${BASE}/config/config.yaml
39
+ sed -i "s/basicAuthMode: false/basicAuthMode: true/" ${BASE}/config/config.yaml
40
+
41
+ # 显示配置文件内容以便验证
42
+ echo "配置文件内容:"
43
+ cat ${BASE}/config/config.yaml
44
+
45
+ # 启动数据同步服务(如果提供了必要的环境变量)
46
+ if [[ ! -z "${HF_TOKEN}" ]] && [[ ! -z "${DATASET_ID}" ]]; then
47
+ echo "启动数据同步服务..."
48
+ nohup ${BASE}/sync_data.sh > ${BASE}/sync_data.log 2>&1 &
49
+ echo "数据同步服务已在后台启动"
50
+ else
51
+ echo "未提供HF_TOKEN或DATASET_ID,不启动数据同步服务"
52
+ fi
53
+
54
+ # 正常启动服务器
55
+ exec node server.js --listen "$@"
sync_data.sh ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # 检查环境变量
4
+ if [[ -z "$HF_TOKEN" ]] || [[ -z "$DATASET_ID" ]]; then
5
+ echo "未启用备份功能 - 缺少HF_TOKEN或DATASET_ID环境变量"
6
+ exit 0
7
+ fi
8
+
9
+ # 创建临时目录
10
+ TEMP_DIR="/tmp/sillytavern_backup"
11
+ mkdir -p $TEMP_DIR
12
+
13
+ # 安装python和huggingface_hub
14
+ if ! command -v python3 &> /dev/null; then
15
+ echo "正在安装Python..."
16
+ apk add --no-cache python3 py3-pip
17
+ fi
18
+
19
+ if ! python3 -c "import huggingface_hub" &> /dev/null; then
20
+ echo "正在安装huggingface_hub..."
21
+ pip3 install --no-cache-dir huggingface_hub
22
+ fi
23
+
24
+ # 上传备份
25
+ upload_backup() {
26
+ file_path="$1"
27
+ file_name="$2"
28
+ token="$HF_TOKEN"
29
+ repo_id="$DATASET_ID"
30
+
31
+ python3 -c "
32
+ from huggingface_hub import HfApi
33
+ import sys
34
+ import os
35
+ def manage_backups(api, repo_id, max_files=10):
36
+ files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')
37
+ backup_files = [f for f in files if f.startswith('sillytavern_backup_') and f.endswith('.tar.gz')]
38
+ backup_files.sort()
39
+
40
+ if len(backup_files) >= max_files:
41
+ files_to_delete = backup_files[:(len(backup_files) - max_files + 1)]
42
+ for file_to_delete in files_to_delete:
43
+ try:
44
+ api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type='dataset')
45
+ print(f'已删除旧备份: {file_to_delete}')
46
+ except Exception as e:
47
+ print(f'删除 {file_to_delete} 时出错: {str(e)}')
48
+ api = HfApi(token='$token')
49
+ try:
50
+ api.upload_file(
51
+ path_or_fileobj='$file_path',
52
+ path_in_repo='$file_name',
53
+ repo_id='$repo_id',
54
+ repo_type='dataset'
55
+ )
56
+ print(f'成功上传 $file_name')
57
+
58
+ manage_backups(api, '$repo_id')
59
+ except Exception as e:
60
+ print(f'上传文件时出错: {str(e)}')
61
+ "
62
+ }
63
+
64
+ # 下载最新备份
65
+ download_latest_backup() {
66
+ token="$HF_TOKEN"
67
+ repo_id="$DATASET_ID"
68
+ data_dir="/home/node/app/data"
69
+
70
+ # 确保数据目录存在
71
+ mkdir -p $data_dir
72
+
73
+ python3 -c "
74
+ from huggingface_hub import HfApi
75
+ import sys
76
+ import os
77
+ import tarfile
78
+ import tempfile
79
+ api = HfApi(token='$token')
80
+ try:
81
+ files = api.list_repo_files(repo_id='$repo_id', repo_type='dataset')
82
+ backup_files = [f for f in files if f.startswith('sillytavern_backup_') and f.endswith('.tar.gz')]
83
+
84
+ if not backup_files:
85
+ print('未找到备份文件')
86
+ sys.exit()
87
+
88
+ latest_backup = sorted(backup_files)[-1]
89
+
90
+ with tempfile.TemporaryDirectory() as temp_dir:
91
+ filepath = api.hf_hub_download(
92
+ repo_id='$repo_id',
93
+ filename=latest_backup,
94
+ repo_type='dataset',
95
+ local_dir=temp_dir
96
+ )
97
+
98
+ if filepath and os.path.exists(filepath):
99
+ with tarfile.open(filepath, 'r:gz') as tar:
100
+ tar.extractall('$data_dir')
101
+ print(f'成功从 {latest_backup} 恢复备份')
102
+
103
+ except Exception as e:
104
+ print(f'下载备份时出错: {str(e)}')
105
+ "
106
+ }
107
+
108
+ # 首次启动时下载最新备份
109
+ echo "正在从HuggingFace下载最新备份..."
110
+ download_latest_backup
111
+
112
+ # 同步函数
113
+ sync_data() {
114
+ while true; do
115
+ echo "开始同步进程,时间: $(date)"
116
+
117
+ data_dir="/home/node/app/data"
118
+ if [ -d "$data_dir" ]; then
119
+ timestamp=$(date +%Y%m%d_%H%M%S)
120
+ backup_file="sillytavern_backup_${timestamp}.tar.gz"
121
+
122
+ # 压缩数据目录
123
+ tar -czf "${TEMP_DIR}/${backup_file}" -C "$data_dir" .
124
+
125
+ echo "正在上传备份到HuggingFace..."
126
+ upload_backup "${TEMP_DIR}/${backup_file}" "${backup_file}"
127
+
128
+ rm -f "${TEMP_DIR}/${backup_file}"
129
+ else
130
+ echo "数据目录尚不存在,等待下次同步..."
131
+ fi
132
+
133
+ SYNC_INTERVAL=${SYNC_INTERVAL:-3600}
134
+ echo "下次同步将在 ${SYNC_INTERVAL} 秒后进行..."
135
+ sleep $SYNC_INTERVAL
136
+ done
137
+ }
138
+
139
+ # 启动同步进程
140
+ sync_data