hxger commited on
Commit
8a8c15b
·
verified ·
1 Parent(s): dbe641a

Upload 3 files

Browse files
Files changed (3) hide show
  1. Dockerfile +23 -22
  2. README.md +11 -11
  3. sync_data.sh +127 -123
Dockerfile CHANGED
@@ -1,22 +1,23 @@
1
- FROM chaoszhu/easynode AS base
2
-
3
- RUN apk add --no-cache python3 python3-dev py3-pip git
4
-
5
- RUN mkdir -p /easynode/app/logs && \
6
- mkdir -p /easynode/app/db && \
7
- chown -R node:node /easynode/app && \
8
- chmod -R 755 /easynode/app
9
-
10
- ENV VIRTUAL_ENV=/easynode/venv
11
- RUN python3 -m venv $VIRTUAL_ENV
12
- ENV PATH="$VIRTUAL_ENV/bin:$PATH"
13
-
14
- RUN pip install --no-cache-dir huggingface_hub
15
-
16
- USER node
17
- WORKDIR /easynode/app
18
-
19
- COPY --chown=node:node sync_data.sh /easynode/app/
20
- RUN chmod +x /easynode/app/sync_data.sh
21
-
22
- CMD ["/bin/sh", "-c", "./sync_data.sh & npm run start"]
 
 
1
+ FROM chaoszhu/easynode AS base
2
+
3
+ RUN apk add --no-cache python3 python3-dev py3-pip git
4
+
5
+ RUN mkdir -p /easynode/app/logs && \
6
+ mkdir -p /easynode/app/db && \
7
+ chown -R node:node /easynode/app && \
8
+ chmod -R 755 /easynode/app
9
+
10
+ ENV VIRTUAL_ENV=/easynode/venv
11
+ RUN python3 -m venv $VIRTUAL_ENV
12
+ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
13
+ ENV PORT=8083
14
+
15
+ RUN pip install --no-cache-dir huggingface_hub
16
+
17
+ USER node
18
+ WORKDIR /easynode/app
19
+
20
+ COPY --chown=node:node sync_data.sh /easynode/app/
21
+ RUN chmod +x /easynode/app/sync_data.sh
22
+
23
+ CMD ["/bin/sh", "-c", "./sync_data.sh & npm run start"]
README.md CHANGED
@@ -1,11 +1,11 @@
1
- ---
2
- title: Wanlian
3
- emoji: 📉
4
- colorFrom: yellow
5
- colorTo: gray
6
- sdk: docker
7
- pinned: false
8
- app_port: 8000
9
- ---
10
-
11
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
+ ---
2
+ title: Wanlian
3
+ emoji: 📉
4
+ colorFrom: yellow
5
+ colorTo: gray
6
+ sdk: docker
7
+ pinned: false
8
+ app_port: 8083
9
+ ---
10
+
11
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
sync_data.sh CHANGED
@@ -1,124 +1,128 @@
1
- #!/bin/sh
2
-
3
- # 检查环境变量
4
- if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
5
- echo "Starting without backup functionality - missing HF_TOKEN or DATASET_ID"
6
- exit 1
7
- fi
8
-
9
- # 激活虚拟环境
10
- . /easynode/venv/bin/activate
11
-
12
- # 上传备份
13
- cat > /easynode/app/hf_sync.py << 'EOL'
14
- from huggingface_hub import HfApi
15
- import sys
16
- import os
17
- import tarfile
18
- import tempfile
19
-
20
- def manage_backups(api, repo_id, max_files=50):
21
- files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
22
- backup_files = [f for f in files if f.startswith('easynode_backup_') and f.endswith('.tar.gz')]
23
- backup_files.sort()
24
-
25
- if len(backup_files) >= max_files:
26
- files_to_delete = backup_files[:(len(backup_files) - max_files + 1)]
27
- for file_to_delete in files_to_delete:
28
- try:
29
- api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type="dataset")
30
- print(f'Deleted old backup: {file_to_delete}')
31
- except Exception as e:
32
- print(f'Error deleting {file_to_delete}: {str(e)}')
33
-
34
- def upload_backup(file_path, file_name, token, repo_id):
35
- api = HfApi(token=token)
36
- try:
37
- api.upload_file(
38
- path_or_fileobj=file_path,
39
- path_in_repo=file_name,
40
- repo_id=repo_id,
41
- repo_type="dataset"
42
- )
43
- print(f"Successfully uploaded {file_name}")
44
-
45
- manage_backups(api, repo_id)
46
- except Exception as e:
47
- print(f"Error uploading file: {str(e)}")
48
-
49
- # 下载最新备份
50
- def download_latest_backup(token, repo_id):
51
- try:
52
- api = HfApi(token=token)
53
- files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
54
- backup_files = [f for f in files if f.startswith('easynode_backup_') and f.endswith('.tar.gz')]
55
-
56
- if not backup_files:
57
- print("No backup files found")
58
- return
59
-
60
- latest_backup = sorted(backup_files)[-1]
61
-
62
- with tempfile.TemporaryDirectory() as temp_dir:
63
- filepath = api.hf_hub_download(
64
- repo_id=repo_id,
65
- filename=latest_backup,
66
- repo_type="dataset",
67
- local_dir=temp_dir
68
- )
69
-
70
- if filepath and os.path.exists(filepath):
71
- with tarfile.open(filepath, 'r:gz') as tar:
72
- tar.extractall('/easynode/app/')
73
- print(f"Successfully restored backup from {latest_backup}")
74
-
75
- except Exception as e:
76
- print(f"Error downloading backup: {str(e)}")
77
-
78
- if __name__ == "__main__":
79
- action = sys.argv[1]
80
- token = sys.argv[2]
81
- repo_id = sys.argv[3]
82
-
83
- if action == "upload":
84
- file_path = sys.argv[4]
85
- file_name = sys.argv[5]
86
- upload_backup(file_path, file_name, token, repo_id)
87
- elif action == "download":
88
- download_latest_backup(token, repo_id)
89
- EOL
90
-
91
- # 首次启动时下载最新备份
92
- echo "Downloading latest backup from HuggingFace..."
93
- python hf_sync.py download "${HF_TOKEN}" "${DATASET_ID}"
94
-
95
- # 同步函数
96
- sync_data() {
97
- while true; do
98
- echo "Starting sync process at $(date)"
99
-
100
- if [ -d "/easynode/app/db" ]; then
101
- timestamp=$(date +%Y%m%d_%H%M%S)
102
- backup_file="easynode_backup_${timestamp}.tar.gz"
103
-
104
- tar -czf "/tmp/${backup_file}" db/
105
-
106
- echo "Uploading backup to HuggingFace..."
107
- python hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
108
-
109
- rm -f "/tmp/${backup_file}"
110
- else
111
- echo "Data directory does not exist yet, waiting for next sync..."
112
- fi
113
-
114
- SYNC_INTERVAL=${SYNC_INTERVAL:-7200}
115
- echo "Next sync in ${SYNC_INTERVAL} seconds..."
116
- sleep $SYNC_INTERVAL
117
- done
118
- }
119
-
120
- # 启动同步进程
121
- sync_data &
122
-
123
- # 启动主应用
 
 
 
 
124
  exec npm run start
 
1
+ #!/bin/sh
2
+
3
+ # 检查环境变量
4
+ if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
5
+ echo "Starting without backup functionality - missing HF_TOKEN or DATASET_ID"
6
+ exit 1
7
+ fi
8
+
9
+ # 激活虚拟环境
10
+ . /easynode/venv/bin/activate
11
+
12
+ # 上传备份
13
+ cat > /easynode/app/hf_sync.py << 'EOL'
14
+ from huggingface_hub import HfApi
15
+ import sys
16
+ import os
17
+ import tarfile
18
+ import tempfile
19
+
20
+ def manage_backups(api, repo_id, max_files=50):
21
+ files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
22
+ backup_files = [f for f in files if f.startswith('easynode_backup_') and f.endswith('.tar.gz')]
23
+ backup_files.sort()
24
+
25
+ if len(backup_files) >= max_files:
26
+ files_to_delete = backup_files[:(len(backup_files) - max_files + 1)]
27
+ for file_to_delete in files_to_delete:
28
+ try:
29
+ api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type="dataset")
30
+ print(f'Deleted old backup: {file_to_delete}')
31
+ except Exception as e:
32
+ print(f'Error deleting {file_to_delete}: {str(e)}')
33
+
34
+ def upload_backup(file_path, file_name, token, repo_id):
35
+ api = HfApi(token=token)
36
+ try:
37
+ api.upload_file(
38
+ path_or_fileobj=file_path,
39
+ path_in_repo=file_name,
40
+ repo_id=repo_id,
41
+ repo_type="dataset"
42
+ )
43
+ print(f"Successfully uploaded {file_name}")
44
+
45
+ manage_backups(api, repo_id)
46
+ except Exception as e:
47
+ print(f"Error uploading file: {str(e)}")
48
+
49
+ # 下载最新备份
50
+ def download_latest_backup(token, repo_id):
51
+ try:
52
+ api = HfApi(token=token)
53
+ files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
54
+ backup_files = [f for f in files if f.startswith('easynode_backup_') and f.endswith('.tar.gz')]
55
+
56
+ if not backup_files:
57
+ print("No backup files found")
58
+ return
59
+
60
+ latest_backup = sorted(backup_files)[-1]
61
+
62
+ with tempfile.TemporaryDirectory() as temp_dir:
63
+ filepath = api.hf_hub_download(
64
+ repo_id=repo_id,
65
+ filename=latest_backup,
66
+ repo_type="dataset",
67
+ local_dir=temp_dir
68
+ )
69
+
70
+ if filepath and os.path.exists(filepath):
71
+ with tarfile.open(filepath, 'r:gz') as tar:
72
+ tar.extractall('/easynode/app/')
73
+ print(f"Successfully restored backup from {latest_backup}")
74
+
75
+ except Exception as e:
76
+ print(f"Error downloading backup: {str(e)}")
77
+
78
+ if __name__ == "__main__":
79
+ action = sys.argv[1]
80
+ token = sys.argv[2]
81
+ repo_id = sys.argv[3]
82
+
83
+ if action == "upload":
84
+ file_path = sys.argv[4]
85
+ file_name = sys.argv[5]
86
+ upload_backup(file_path, file_name, token, repo_id)
87
+ elif action == "download":
88
+ download_latest_backup(token, repo_id)
89
+ EOL
90
+
91
+ # 首次启动时下载最新备份
92
+ echo "Downloading latest backup from HuggingFace..."
93
+ python hf_sync.py download "${HF_TOKEN}" "${DATASET_ID}"
94
+
95
+ # 同步函数
96
+ sync_data() {
97
+ while true; do
98
+ echo "Starting sync process at $(date)"
99
+
100
+ if [ -d "/easynode/app/db" ]; then
101
+ timestamp=$(date +%Y%m%d_%H%M%S)
102
+ backup_file="easynode_backup_${timestamp}.tar.gz"
103
+
104
+ tar -czf "/tmp/${backup_file}" db/
105
+
106
+ echo "Uploading backup to HuggingFace..."
107
+ python hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
108
+
109
+ rm -f "/tmp/${backup_file}"
110
+ else
111
+ echo "Data directory does not exist yet, waiting for next sync..."
112
+ fi
113
+
114
+ SYNC_INTERVAL=${SYNC_INTERVAL:-7200}
115
+ echo "Next sync in ${SYNC_INTERVAL} seconds..."
116
+ sleep $SYNC_INTERVAL
117
+ done
118
+ }
119
+
120
+ # 启动同步进程
121
+ sync_data &
122
+
123
+ # 确保使用正确的端口
124
+ export PORT=${PORT:-8083}
125
+ echo "Starting application on port: $PORT"
126
+
127
+ # 启动主应用
128
  exec npm run start