HoneyTian commited on
Commit
8bea69a
·
1 Parent(s): 20323b4

add concat

Browse files
.gitignore CHANGED
@@ -12,3 +12,4 @@
12
  **/__pycache__/
13
 
14
  #**/*.wav
 
 
12
  **/__pycache__/
13
 
14
  #**/*.wav
15
+ **/*.xlsx
data/examples/concat/chinese/chinese_1.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d78707b240dc3ced96be02cafbc95cc0bb06e2bae2344f588ab70ff90db990e2
3
+ size 49574
data/examples/concat/chinese/chinese_1_trim.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:97928009f0873c408b9006183f532d00585f458ac40a6a5d32e5e6e96e5ab0ab
3
+ size 36556
data/examples/concat/chinese/chinese_2.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bccddbd8e63bf24d7748dc6379b82f71286b6c375ba84502e1653b7e923c0c83
3
+ size 19274
data/examples/concat/chinese/chinese_2_tts.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4498d2360d5a600cc1d3a10887b8ca058dea923c402121b16a276b37607e5bc5
3
+ size 35100
data/examples/concat/chinese/chinese_3.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbe485c141902ee59eda7b110e1e2e90ee470e9069c7e943a174dd287c141e77
3
+ size 32826
data/examples/concat/chinese/chinese_3_trim.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3e29a8207daaba5507b75fc900e9a1d9aa19007d5b2a6b030fdf149109d4123
3
+ size 18764
data/examples/concat/chinese/chinese_4.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a0cb1a1b790af9a1d32fc8949b32bef8ab3cf2e50dddd2c07d7066c251ed121
3
+ size 18764
data/examples/concat/chinese/chinese_5.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:321c8dd1b44740713a5cae91c2a6bbe4588589ec03161323097768cb976f18fb
3
+ size 103070
data/examples/concat/chinese/chinese_5_trim.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3aaede84ac31ee51be49f7ebf265cb21485e8d5dd913750ab1a6dbed0192b566
3
+ size 89132
data/examples/concat/english/English_1.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:359a461350c4d00b398d7c777c43543935faa510707bd4579b9a1fd29898081e
3
+ size 312208
data/examples/concat/english/English_2.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1352b7fc7c10abd20de3ee9528da5513a460ed37e5c93f6726c052e94c3e94ac
3
+ size 43698
data/examples/concat/english/English_3.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e9dc1d88de3d28f5fac39a5fe6b59ae5bc5f139ee527e9836fe03bc6547964ff
3
+ size 141380
data/examples/concat/english/xtts_v2_english_2_volume_adapt.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:90ea4eeabd7328d774ba4ecb1c4fb55dc9267c91c099c4ca78723c3890d16358
3
+ size 35544
examples/jik_trim/step_1_download_audio.py ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/python3
2
+ # -*- coding: utf-8 -*-
3
+ import argparse
4
+ from urllib.parse import urlparse
5
+
6
+ from pathlib import Path
7
+
8
+ import requests
9
+ import pandas as pd
10
+
11
+ from project_settings import project_path
12
+
13
+
14
+ def get_args():
15
+ parser = argparse.ArgumentParser()
16
+ parser.add_argument(
17
+ "--audio_file",
18
+ default="audio.xlsx",
19
+ type=str,
20
+ )
21
+ parser.add_argument(
22
+ "--output_dir",
23
+ default=(project_path / "temp/audio_trim/origin").as_posix(),
24
+ type=str,
25
+ )
26
+ args = parser.parse_args()
27
+ return args
28
+
29
+
30
+ def main():
31
+ args = get_args()
32
+
33
+ output_dir = Path(args.output_dir)
34
+ output_dir.mkdir(parents=True, exist_ok=True)
35
+
36
+ df = pd.read_excel(args.audio_file)
37
+
38
+ for i, row in df.iterrows():
39
+ name = row["name"]
40
+ scene_id = row["scene_id"]
41
+ audio_id = row["audio_id"]
42
+ audio_url = row["audio_url"]
43
+
44
+ schema = urlparse(audio_url)
45
+ path = schema.path
46
+
47
+ filename = output_dir / path[1:]
48
+ filename.parent.mkdir(parents=True, exist_ok=True)
49
+
50
+ resp = requests.get(audio_url)
51
+ with open(filename.as_posix(), "wb") as f:
52
+ f.write(resp.content)
53
+ return
54
+
55
+
56
+ if __name__ == "__main__":
57
+ main()
examples/jik_trim/step_2_trim_audio.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/python3
2
+ # -*- coding: utf-8 -*-
3
+ import argparse
4
+ import shutil
5
+ from urllib.parse import urlparse
6
+ import json
7
+
8
+ import librosa
9
+ import numpy as np
10
+ from pathlib import Path
11
+
12
+ from gradio_client import Client, handle_file
13
+ import requests
14
+ import pandas as pd
15
+ from scipy.io import wavfile
16
+
17
+ from project_settings import project_path
18
+
19
+
20
+ def get_args():
21
+ parser = argparse.ArgumentParser()
22
+ parser.add_argument(
23
+ "--audio_file",
24
+ default="audio.xlsx",
25
+ type=str,
26
+ )
27
+ parser.add_argument(
28
+ "--audio_dir",
29
+ default=(project_path / "temp/audio_trim/origin").as_posix(),
30
+ type=str,
31
+ )
32
+ parser.add_argument(
33
+ "--output_dir",
34
+ default=(project_path / "temp/audio_trim/trimmed").as_posix(),
35
+ type=str,
36
+ )
37
+ args = parser.parse_args()
38
+ return args
39
+
40
+
41
+ client = Client("http://127.0.0.1:7861/")
42
+
43
+
44
+ def main():
45
+ args = get_args()
46
+
47
+ audio_dir = Path(args.audio_dir)
48
+ audio_dir.mkdir(parents=True, exist_ok=True)
49
+
50
+ output_dir = Path(args.output_dir)
51
+ output_dir.mkdir(parents=True, exist_ok=True)
52
+
53
+ df = pd.read_excel(args.audio_file)
54
+
55
+ for i, row in df.iterrows():
56
+ name = row["name"]
57
+ scene_id = row["scene_id"]
58
+ audio_id = row["audio_id"]
59
+ audio_url = row["audio_url"]
60
+
61
+ schema = urlparse(audio_url)
62
+ path = schema.path
63
+
64
+ filename = audio_dir / path[1:]
65
+
66
+ kwargs = {
67
+ "silence_threshold": -40,
68
+ "min_silence_len": 200,
69
+ "min_kept_silence": 200,
70
+ "mode": "trim"
71
+ }
72
+ kwargs = json.dumps(kwargs, ensure_ascii=False, indent=4)
73
+ output_audio, log = client.predict(
74
+ audio_t=handle_file(filename.as_posix()),
75
+ kwargs=kwargs,
76
+ engine="pydub_scipy",
77
+ api_name="/when_click_trim_audio"
78
+ )
79
+
80
+ trimmed_filename = output_dir / path[1:]
81
+ trimmed_filename.parent.mkdir(parents=True, exist_ok=True)
82
+
83
+ shutil.move(
84
+ output_audio,
85
+ trimmed_filename.as_posix()
86
+ )
87
+ return
88
+
89
+
90
+ if __name__ == "__main__":
91
+ main()
examples/jik_trim/step_3_upload_to_obs.py ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/python3
2
+ # -*- coding: utf-8 -*-
3
+ import argparse
4
+ import shutil
5
+ from urllib.parse import urlparse
6
+ import json
7
+
8
+ import librosa
9
+ import numpy as np
10
+ from pathlib import Path
11
+
12
+ from gradio_client import Client, handle_file
13
+ import requests
14
+ import pandas as pd
15
+ from scipy.io import wavfile
16
+
17
+ from project_settings import project_path, environment
18
+
19
+
20
+ def get_args():
21
+ parser = argparse.ArgumentParser()
22
+ parser.add_argument(
23
+ "--audio_file",
24
+ default="audio.xlsx",
25
+ type=str,
26
+ )
27
+ parser.add_argument(
28
+ "--audio_dir",
29
+ default=(project_path / "temp/audio_trim/trimmed").as_posix(),
30
+ type=str,
31
+ )
32
+ parser.add_argument(
33
+ "--obs_secret_id",
34
+ default=environment.get("obs_secret_id"),
35
+ type=str,
36
+ )
37
+ parser.add_argument(
38
+ "--obs_secret_key",
39
+ default=environment.get("obs_secret_key"),
40
+ type=str,
41
+ )
42
+ args = parser.parse_args()
43
+ return args
44
+
45
+
46
+ client = Client("http://127.0.0.1:7861/")
47
+
48
+
49
+ def main():
50
+ args = get_args()
51
+
52
+ obs_kwargs = {
53
+ "obs_supplier": "tencent",
54
+ "url_prefix": "https://nxai-hk-1259196162.cos.ap-hongkong.myqcloud.com",
55
+ "endpoint_url": "https://cos.ap-hongkong.myqcloud.com",
56
+ "region": "ap-hongkong",
57
+ "secret_id": args.obs_secret_id,
58
+ "secret_key": args.obs_secret_key,
59
+ "bucket": "nxai-hk-1259196162",
60
+ # "debug": True
61
+ }
62
+ obs_kwargs = json.dumps(obs_kwargs, ensure_ascii=False, indent=4)
63
+ print(obs_kwargs)
64
+
65
+ audio_dir = Path(args.audio_dir)
66
+ audio_dir.mkdir(parents=True, exist_ok=True)
67
+
68
+ df = pd.read_excel(args.audio_file)
69
+
70
+ for i, row in df.iterrows():
71
+ name = row["name"]
72
+ scene_id = row["scene_id"]
73
+ audio_id = row["audio_id"]
74
+ audio_url = row["audio_url"]
75
+
76
+ schema = urlparse(audio_url)
77
+ url_path = schema.path
78
+
79
+ filename = audio_dir / url_path[1:]
80
+
81
+ file_url, message = client.predict(
82
+ filename=handle_file(filename.as_posix()),
83
+ url_path=url_path[1:],
84
+ obs_kwargs=obs_kwargs,
85
+ api_name="/when_click_upload_to_obs"
86
+ )
87
+ print(filename.as_posix())
88
+ print(f"file_url1: {audio_url}")
89
+ print(f"file_url2: {file_url}")
90
+ print(f"message: {message}")
91
+ print("-" * 150)
92
+
93
+ return
94
+
95
+
96
+ if __name__ == "__main__":
97
+ main()
log.py ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/python3
2
+ # -*- coding: utf-8 -*-
3
+ import logging
4
+ from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
5
+ import os
6
+
7
+
8
+ def setup_size_rotating(log_directory: str):
9
+ fmt = "%(asctime)s - %(name)s - %(levelname)s %(filename)s:%(lineno)d > %(message)s"
10
+
11
+ stream_handler = logging.StreamHandler()
12
+ stream_handler.setLevel(logging.INFO)
13
+ stream_handler.setFormatter(logging.Formatter(fmt))
14
+
15
+ # main
16
+ main_logger = logging.getLogger("main")
17
+ main_logger.addHandler(stream_handler)
18
+ main_info_file_handler = RotatingFileHandler(
19
+ filename=os.path.join(log_directory, "main.log"),
20
+ maxBytes=100*1024*1024, # 100MB
21
+ encoding="utf-8",
22
+ backupCount=2,
23
+ )
24
+ main_info_file_handler.setLevel(logging.INFO)
25
+ main_info_file_handler.setFormatter(logging.Formatter(fmt))
26
+ main_logger.addHandler(main_info_file_handler)
27
+
28
+ # http
29
+ http_logger = logging.getLogger("http")
30
+ http_file_handler = RotatingFileHandler(
31
+ filename=os.path.join(log_directory, "http.log"),
32
+ maxBytes=100*1024*1024, # 100MB
33
+ encoding="utf-8",
34
+ backupCount=2,
35
+ )
36
+ http_file_handler.setLevel(logging.DEBUG)
37
+ http_file_handler.setFormatter(logging.Formatter(fmt))
38
+ http_logger.addHandler(http_file_handler)
39
+
40
+ # api
41
+ api_logger = logging.getLogger("api")
42
+ api_file_handler = RotatingFileHandler(
43
+ filename=os.path.join(log_directory, "api.log"),
44
+ maxBytes=10*1024*1024, # 10MB
45
+ encoding="utf-8",
46
+ backupCount=2,
47
+ )
48
+ api_file_handler.setLevel(logging.DEBUG)
49
+ api_file_handler.setFormatter(logging.Formatter(fmt))
50
+ api_logger.addHandler(api_file_handler)
51
+
52
+ # toolbox
53
+ toolbox_logger = logging.getLogger("toolbox")
54
+ toolbox_logger.addHandler(stream_handler)
55
+ toolbox_file_handler = RotatingFileHandler(
56
+ filename=os.path.join(log_directory, "toolbox.log"),
57
+ maxBytes=10*1024*1024, # 10MB
58
+ encoding="utf-8",
59
+ backupCount=2,
60
+ )
61
+ toolbox_file_handler.setLevel(logging.DEBUG)
62
+ toolbox_file_handler.setFormatter(logging.Formatter(fmt))
63
+ toolbox_logger.addHandler(toolbox_file_handler)
64
+
65
+ # alarm
66
+ alarm_logger = logging.getLogger("alarm")
67
+ alarm_file_handler = RotatingFileHandler(
68
+ filename=os.path.join(log_directory, "alarm.log"),
69
+ maxBytes=1*1024*1024, # 1MB
70
+ encoding="utf-8",
71
+ backupCount=2,
72
+ )
73
+ alarm_file_handler.setLevel(logging.DEBUG)
74
+ alarm_file_handler.setFormatter(logging.Formatter(fmt))
75
+ alarm_logger.addHandler(alarm_file_handler)
76
+
77
+ debug_file_handler = RotatingFileHandler(
78
+ filename=os.path.join(log_directory, "debug.log"),
79
+ maxBytes=1*1024*1024, # 1MB
80
+ encoding="utf-8",
81
+ backupCount=2,
82
+ )
83
+ debug_file_handler.setLevel(logging.DEBUG)
84
+ debug_file_handler.setFormatter(logging.Formatter(fmt))
85
+
86
+ info_file_handler = RotatingFileHandler(
87
+ filename=os.path.join(log_directory, "info.log"),
88
+ maxBytes=1*1024*1024, # 1MB
89
+ encoding="utf-8",
90
+ backupCount=2,
91
+ )
92
+ info_file_handler.setLevel(logging.INFO)
93
+ info_file_handler.setFormatter(logging.Formatter(fmt))
94
+
95
+ error_file_handler = RotatingFileHandler(
96
+ filename=os.path.join(log_directory, "error.log"),
97
+ maxBytes=1*1024*1024, # 1MB
98
+ encoding="utf-8",
99
+ backupCount=2,
100
+ )
101
+ error_file_handler.setLevel(logging.ERROR)
102
+ error_file_handler.setFormatter(logging.Formatter(fmt))
103
+
104
+ logging.basicConfig(
105
+ level=logging.DEBUG,
106
+ datefmt="%a, %d %b %Y %H:%M:%S",
107
+ handlers=[
108
+ debug_file_handler,
109
+ info_file_handler,
110
+ error_file_handler,
111
+ ]
112
+ )
113
+
114
+
115
+ def setup_time_rotating(log_directory: str):
116
+ fmt = "%(asctime)s - %(name)s - %(levelname)s %(filename)s:%(lineno)d > %(message)s"
117
+
118
+ stream_handler = logging.StreamHandler()
119
+ stream_handler.setLevel(logging.INFO)
120
+ stream_handler.setFormatter(logging.Formatter(fmt))
121
+
122
+ # main
123
+ main_logger = logging.getLogger("main")
124
+ main_logger.addHandler(stream_handler)
125
+ main_info_file_handler = TimedRotatingFileHandler(
126
+ filename=os.path.join(log_directory, "main.log"),
127
+ encoding="utf-8",
128
+ when="midnight",
129
+ interval=1,
130
+ backupCount=7
131
+ )
132
+ main_info_file_handler.setLevel(logging.INFO)
133
+ main_info_file_handler.setFormatter(logging.Formatter(fmt))
134
+ main_logger.addHandler(main_info_file_handler)
135
+
136
+ # http
137
+ http_logger = logging.getLogger("http")
138
+ http_file_handler = TimedRotatingFileHandler(
139
+ filename=os.path.join(log_directory, "http.log"),
140
+ encoding='utf-8',
141
+ when="midnight",
142
+ interval=1,
143
+ backupCount=7
144
+ )
145
+ http_file_handler.setLevel(logging.DEBUG)
146
+ http_file_handler.setFormatter(logging.Formatter(fmt))
147
+ http_logger.addHandler(http_file_handler)
148
+
149
+ # api
150
+ api_logger = logging.getLogger("api")
151
+ api_file_handler = TimedRotatingFileHandler(
152
+ filename=os.path.join(log_directory, "api.log"),
153
+ encoding='utf-8',
154
+ when="midnight",
155
+ interval=1,
156
+ backupCount=7
157
+ )
158
+ api_file_handler.setLevel(logging.DEBUG)
159
+ api_file_handler.setFormatter(logging.Formatter(fmt))
160
+ api_logger.addHandler(api_file_handler)
161
+
162
+ # toolbox
163
+ toolbox_logger = logging.getLogger("toolbox")
164
+ toolbox_file_handler = RotatingFileHandler(
165
+ filename=os.path.join(log_directory, "toolbox.log"),
166
+ maxBytes=10*1024*1024, # 10MB
167
+ encoding="utf-8",
168
+ backupCount=2,
169
+ )
170
+ toolbox_file_handler.setLevel(logging.DEBUG)
171
+ toolbox_file_handler.setFormatter(logging.Formatter(fmt))
172
+ toolbox_logger.addHandler(toolbox_file_handler)
173
+
174
+ # alarm
175
+ alarm_logger = logging.getLogger("alarm")
176
+ alarm_file_handler = TimedRotatingFileHandler(
177
+ filename=os.path.join(log_directory, "alarm.log"),
178
+ encoding="utf-8",
179
+ when="midnight",
180
+ interval=1,
181
+ backupCount=7
182
+ )
183
+ alarm_file_handler.setLevel(logging.DEBUG)
184
+ alarm_file_handler.setFormatter(logging.Formatter(fmt))
185
+ alarm_logger.addHandler(alarm_file_handler)
186
+
187
+ debug_file_handler = TimedRotatingFileHandler(
188
+ filename=os.path.join(log_directory, "debug.log"),
189
+ encoding="utf-8",
190
+ when="D",
191
+ interval=1,
192
+ backupCount=7
193
+ )
194
+ debug_file_handler.setLevel(logging.DEBUG)
195
+ debug_file_handler.setFormatter(logging.Formatter(fmt))
196
+
197
+ info_file_handler = TimedRotatingFileHandler(
198
+ filename=os.path.join(log_directory, "info.log"),
199
+ encoding="utf-8",
200
+ when="D",
201
+ interval=1,
202
+ backupCount=7
203
+ )
204
+ info_file_handler.setLevel(logging.INFO)
205
+ info_file_handler.setFormatter(logging.Formatter(fmt))
206
+
207
+ error_file_handler = TimedRotatingFileHandler(
208
+ filename=os.path.join(log_directory, "error.log"),
209
+ encoding="utf-8",
210
+ when="D",
211
+ interval=1,
212
+ backupCount=7
213
+ )
214
+ error_file_handler.setLevel(logging.ERROR)
215
+ error_file_handler.setFormatter(logging.Formatter(fmt))
216
+
217
+ logging.basicConfig(
218
+ level=logging.DEBUG,
219
+ datefmt="%a, %d %b %Y %H:%M:%S",
220
+ handlers=[
221
+ debug_file_handler,
222
+ info_file_handler,
223
+ error_file_handler,
224
+ ]
225
+ )
226
+
227
+
228
+ if __name__ == "__main__":
229
+ pass
main.py CHANGED
@@ -13,7 +13,9 @@ docker run -itd \
13
  audio_edit:v20250314_1357
14
  """
15
  import argparse
 
16
  import json
 
17
  from pathlib import Path
18
  import platform
19
  import tempfile
@@ -24,7 +26,7 @@ import gradio as gr
24
  import numpy as np
25
  from scipy.io import wavfile
26
 
27
- from project_settings import environment, project_path
28
  from toolbox.audio_edit.info import get_audio_info, engine_to_function as info_engine_to_function
29
  from toolbox.audio_edit.convert import audio_convert, engine_to_function as cvt_engine_to_function
30
  from toolbox.audio_edit.speech_speed import change_speech_speed, engine_to_function as speed_engine_to_function
@@ -33,6 +35,12 @@ from toolbox.audio_edit.augment import mix_speech_and_noise
33
  from toolbox.audio_edit.reverb import reverb, engine_to_function as reverb_engine_to_function
34
  from toolbox.os.command import Command
35
  from toolbox.audio_edit.trim import audio_trim, engine_to_function as trim_engine_to_function
 
 
 
 
 
 
36
 
37
 
38
  def get_args():
@@ -192,6 +200,8 @@ def when_click_trim_audio(audio_t, kwargs: str, engine: str):
192
  kwargs = json.loads(kwargs)
193
  output_file, ext = audio_trim(
194
  filename=filename,
 
 
195
  engine=engine,
196
  **kwargs,
197
  )
@@ -204,6 +214,35 @@ def when_click_trim_audio(audio_t, kwargs: str, engine: str):
204
  return output_file, message
205
 
206
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
207
  def when_click_reverb(audio_t, kwargs: str, engine: str):
208
  sample_rate, signal = audio_t
209
 
@@ -258,6 +297,27 @@ def when_click_mix_speech_and_noise(speech_t, noise_t, snr_db: float):
258
  return (sample_rate1, mix_signal), message
259
 
260
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
261
  audio_convert_examples = [
262
  [
263
  (project_path / "data/examples/default/audio_0_2.wav").as_posix(),
@@ -303,6 +363,12 @@ pad_audio_examples = [
303
 
304
 
305
  trim_examples = [
 
 
 
 
 
 
306
  [
307
  (project_path / "data/examples/mix/speech/000f62f5-5b05-4494-a8db-0eaca3ebd871_th-TH_1678353399860.wav").as_posix(),
308
  '{\n "silence_threshold": -40,\n "min_silence_len": 200,\n "min_kept_silence": 200,\n "mode": "trim"\n}',
@@ -311,6 +377,49 @@ trim_examples = [
311
  ]
312
 
313
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
314
  reverb_examples = [
315
  [
316
  (project_path / "data/examples/default/audio_0_2.wav").as_posix(),
@@ -540,7 +649,6 @@ def main():
540
  trim_kwargs = gr.Textbox(lines=8, label="kwargs")
541
  trim_engine = gr.Dropdown(choices=trim_choices, value=trim_choices[0], label="engine")
542
  trim_button = gr.Button(variant="primary")
543
-
544
  with gr.Column(variant="panel", scale=5):
545
  trim_output_audio = gr.Audio(label="output_audio")
546
  trim_log = gr.Text(label="log")
@@ -559,6 +667,29 @@ def main():
559
  trim_output_audio, trim_log
560
  ],
561
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
562
  with gr.TabItem("reverb"):
563
  with gr.Row():
564
  with gr.Column(variant="panel", scale=5):
@@ -613,6 +744,22 @@ def main():
613
  mix_output_audio, mix_log
614
  ],
615
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
616
  with gr.TabItem("shell"):
617
  shell_text = gr.Textbox(label="cmd")
618
  shell_button = gr.Button("run")
 
13
  audio_edit:v20250314_1357
14
  """
15
  import argparse
16
+ import asyncio
17
  import json
18
+ import logging
19
  from pathlib import Path
20
  import platform
21
  import tempfile
 
26
  import numpy as np
27
  from scipy.io import wavfile
28
 
29
+ from project_settings import environment, project_path, log_directory
30
  from toolbox.audio_edit.info import get_audio_info, engine_to_function as info_engine_to_function
31
  from toolbox.audio_edit.convert import audio_convert, engine_to_function as cvt_engine_to_function
32
  from toolbox.audio_edit.speech_speed import change_speech_speed, engine_to_function as speed_engine_to_function
 
35
  from toolbox.audio_edit.reverb import reverb, engine_to_function as reverb_engine_to_function
36
  from toolbox.os.command import Command
37
  from toolbox.audio_edit.trim import audio_trim, engine_to_function as trim_engine_to_function
38
+ from toolbox.obs.obs import OBSManager
39
+ import log
40
+
41
+ log.setup_size_rotating(log_directory=log_directory)
42
+
43
+ logger = logging.getLogger("main")
44
 
45
 
46
  def get_args():
 
200
  kwargs = json.loads(kwargs)
201
  output_file, ext = audio_trim(
202
  filename=filename,
203
+ signal=signal,
204
+ sample_rate=sample_rate,
205
  engine=engine,
206
  **kwargs,
207
  )
 
214
  return output_file, message
215
 
216
 
217
+ def when_click_concat_audio(files: List[str]):
218
+ check_sample_rate = None
219
+ check_dtype = None
220
+
221
+ message = "success"
222
+ try:
223
+ signal_list = list()
224
+ for file in files:
225
+ sample_rate, signal = wavfile.read(file)
226
+ if check_sample_rate is None:
227
+ check_sample_rate = sample_rate
228
+ if check_dtype is None:
229
+ check_dtype = signal.dtype
230
+
231
+ if check_sample_rate != sample_rate:
232
+ raise AssertionError
233
+ if check_dtype != signal.dtype:
234
+ raise AssertionError
235
+
236
+ signal_list.append(signal)
237
+
238
+ signal_list = np.concat(signal_list)
239
+ except Exception as e:
240
+ signal_list = None
241
+ message = f"failed. error type: {type(e)}, error text: {str(e)}"
242
+
243
+ return (check_sample_rate, signal_list), message
244
+
245
+
246
  def when_click_reverb(audio_t, kwargs: str, engine: str):
247
  sample_rate, signal = audio_t
248
 
 
297
  return (sample_rate1, mix_signal), message
298
 
299
 
300
+ def when_click_upload_to_obs(filename: str, url_path: str, obs_kwargs: str):
301
+ message = "success"
302
+
303
+ try:
304
+ obs_kwargs = json.loads(obs_kwargs)
305
+ obs_manager = OBSManager(
306
+ **obs_kwargs,
307
+ )
308
+ file_url = asyncio.run(
309
+ obs_manager.upload_file_to_obs(
310
+ filename, url_path
311
+ )
312
+ )
313
+
314
+ except Exception as e:
315
+ file_url = None
316
+ message = f"failed. error type: {type(e)}, error text: {str(e)}"
317
+
318
+ return file_url, message
319
+
320
+
321
  audio_convert_examples = [
322
  [
323
  (project_path / "data/examples/default/audio_0_2.wav").as_posix(),
 
363
 
364
 
365
  trim_examples = [
366
+ [
367
+ (
368
+ project_path / "data/examples/mix/speech/000f62f5-5b05-4494-a8db-0eaca3ebd871_th-TH_1678353399860.wav").as_posix(),
369
+ '{\n "silence_threshold": -40,\n "min_silence_len": 200,\n "min_kept_silence": 200,\n "mode": "trim"\n}',
370
+ "pydub_scipy",
371
+ ],
372
  [
373
  (project_path / "data/examples/mix/speech/000f62f5-5b05-4494-a8db-0eaca3ebd871_th-TH_1678353399860.wav").as_posix(),
374
  '{\n "silence_threshold": -40,\n "min_silence_len": 200,\n "min_kept_silence": 200,\n "mode": "trim"\n}',
 
377
  ]
378
 
379
 
380
+ concat_examples = [
381
+ [
382
+ [
383
+ (project_path / "data/examples/concat/chinese/chinese_1.wav").as_posix(),
384
+ (project_path / "data/examples/concat/chinese/chinese_2_tts.wav").as_posix(),
385
+ (project_path / "data/examples/concat/chinese/chinese_3.wav").as_posix(),
386
+ (project_path / "data/examples/concat/chinese/chinese_4.wav").as_posix(),
387
+ (project_path / "data/examples/concat/chinese/chinese_5.wav").as_posix(),
388
+ ],
389
+ ],
390
+ [
391
+ [
392
+ (project_path / "data/examples/concat/chinese/chinese_1_trim.wav").as_posix(),
393
+ (project_path / "data/examples/concat/chinese/chinese_2_tts.wav").as_posix(),
394
+ (project_path / "data/examples/concat/chinese/chinese_3_trim.wav").as_posix(),
395
+ (project_path / "data/examples/concat/chinese/chinese_4.wav").as_posix(),
396
+ (project_path / "data/examples/concat/chinese/chinese_5_trim.wav").as_posix(),
397
+ ],
398
+ ],
399
+ [
400
+ [
401
+ (project_path / "data/examples/concat/english/English_1.wav").as_posix(),
402
+ (project_path / "data/examples/concat/english/xtts_v2_english_2_volume_adapt.wav").as_posix(),
403
+ (project_path / "data/examples/concat/english/English_3.wav").as_posix(),
404
+ ],
405
+ ],
406
+ [
407
+ [
408
+ (project_path / "data/examples/concat/english/English_1.wav").as_posix(),
409
+ (project_path / "data/examples/concat/english/English_2.wav").as_posix(),
410
+ (project_path / "data/examples/concat/english/English_3.wav").as_posix(),
411
+ ],
412
+ ],
413
+ [
414
+ [
415
+ (project_path / "data/examples/concat/english/English_1.wav").as_posix(),
416
+ (project_path / "data/examples/concat/english/xtts_v2_english_2_volume_adapt.wav").as_posix(),
417
+ (project_path / "data/examples/concat/english/English_3.wav").as_posix(),
418
+ ],
419
+ ],
420
+ ]
421
+
422
+
423
  reverb_examples = [
424
  [
425
  (project_path / "data/examples/default/audio_0_2.wav").as_posix(),
 
649
  trim_kwargs = gr.Textbox(lines=8, label="kwargs")
650
  trim_engine = gr.Dropdown(choices=trim_choices, value=trim_choices[0], label="engine")
651
  trim_button = gr.Button(variant="primary")
 
652
  with gr.Column(variant="panel", scale=5):
653
  trim_output_audio = gr.Audio(label="output_audio")
654
  trim_log = gr.Text(label="log")
 
667
  trim_output_audio, trim_log
668
  ],
669
  )
670
+ with gr.TabItem("concat"):
671
+ with gr.Row():
672
+ with gr.Column(variant="panel", scale=5):
673
+ concat_wav_files = gr.Files(label="wav files")
674
+ concat_button = gr.Button(variant="primary")
675
+ with gr.Column(variant="panel", scale=5):
676
+ concat_output_audio = gr.Audio(label="output_audio")
677
+ concat_log = gr.Text(label="log")
678
+ gr.Examples(
679
+ examples=concat_examples,
680
+ inputs=[concat_wav_files],
681
+ outputs=[
682
+ concat_output_audio, concat_log
683
+ ],
684
+ fn=when_click_concat_audio,
685
+ )
686
+ concat_button.click(
687
+ when_click_concat_audio,
688
+ inputs=[concat_wav_files],
689
+ outputs=[
690
+ concat_output_audio, concat_log
691
+ ],
692
+ )
693
  with gr.TabItem("reverb"):
694
  with gr.Row():
695
  with gr.Column(variant="panel", scale=5):
 
744
  mix_output_audio, mix_log
745
  ],
746
  )
747
+ with gr.TabItem("obs"):
748
+ with gr.Row():
749
+ with gr.Column(variant="panel", scale=5):
750
+ obs_file = gr.File(label="file")
751
+ obs_url_path = gr.Text(label="url_path")
752
+ obs_kwargs = gr.Textbox(label="obs_kwargs")
753
+
754
+ obs_button = gr.Button(variant="primary")
755
+ with gr.Column(variant="panel", scale=5):
756
+ obs_file_url = gr.Text(label="obs_file_url")
757
+ info_log = gr.Text(label="log")
758
+ obs_button.click(
759
+ when_click_upload_to_obs,
760
+ inputs=[obs_file, obs_url_path, obs_kwargs],
761
+ outputs=[obs_file_url, info_log],
762
+ )
763
  with gr.TabItem("shell"):
764
  shell_text = gr.Textbox(label="cmd")
765
  shell_button = gr.Button("run")
project_settings.py CHANGED
@@ -8,6 +8,9 @@ from toolbox.os.environment import EnvironmentManager
8
  project_path = os.path.abspath(os.path.dirname(__file__))
9
  project_path = Path(project_path)
10
 
 
 
 
11
  environment = EnvironmentManager(
12
  path=os.path.join(project_path, "dotenv"),
13
  env=os.environ.get("environment", "dev"),
 
8
  project_path = os.path.abspath(os.path.dirname(__file__))
9
  project_path = Path(project_path)
10
 
11
+ log_directory = project_path / "logs"
12
+ log_directory.mkdir(parents=True, exist_ok=True)
13
+
14
  environment = EnvironmentManager(
15
  path=os.path.join(project_path, "dotenv"),
16
  env=os.environ.get("environment", "dev"),
requirements.txt CHANGED
@@ -8,3 +8,8 @@ tinytag==2.0.0
8
  pedalboard==0.9.16
9
  pyroomacoustics==0.8.3
10
  python-dotenv==1.0.1
 
 
 
 
 
 
8
  pedalboard==0.9.16
9
  pyroomacoustics==0.8.3
10
  python-dotenv==1.0.1
11
+ pandas==2.2.3
12
+ openpyxl==3.1.5
13
+ aiobotocore==2.12.1
14
+ oss2==2.19.1
15
+ tenacity==8.3.0
toolbox/audio_edit/trim.py CHANGED
@@ -6,6 +6,7 @@ import tempfile
6
  import uuid
7
 
8
  import librosa
 
9
  from pydub import AudioSegment
10
  from pydub.silence import detect_silence
11
  from scipy.io import wavfile
@@ -24,12 +25,79 @@ def get_args():
24
  return args
25
 
26
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  def audio_trim_by_pydub(filename: str,
 
 
28
  silence_threshold: int = -40,
29
  min_silence_len: float = 1000,
30
  min_kept_silence: float = 200,
31
  mode: str = "trim"
32
  ):
 
 
 
33
  audio = AudioSegment.from_file(filename, format="wav")
34
  length = len(audio)
35
 
@@ -78,18 +146,19 @@ def audio_trim_by_pydub(filename: str,
78
 
79
 
80
  def audio_trim_by_librosa(filename: str,
81
- sample_rate: int = None,
 
82
  top_db: float = 60,
83
  frame_length: int = 2048,
84
  hop_length: int = 512,
85
  mode: str = "trim",
86
  **kwargs
87
  ):
88
- signal, sample_rate = librosa.load(filename, sr=sample_rate, mono=False)
89
  length = len(signal)
90
 
91
  _, index= librosa.effects.trim(
92
- signal,
93
  top_db=top_db, frame_length=frame_length,
94
  hop_length=hop_length,
95
  **kwargs
@@ -126,17 +195,21 @@ def audio_trim_by_librosa(filename: str,
126
 
127
 
128
  engine_to_function = {
 
129
  "pydub": audio_trim_by_pydub,
130
  "librosa": audio_trim_by_librosa,
131
  }
132
 
133
 
134
- def audio_trim(filename: str, engine: str = "librosa", **kwargs):
 
 
 
135
  function = engine_to_function.get(engine)
136
  if function is None:
137
  raise AssertionError(f"invalid engine: {engine}")
138
 
139
- return function(filename, **kwargs)
140
 
141
 
142
  def main():
 
6
  import uuid
7
 
8
  import librosa
9
+ import numpy as np
10
  from pydub import AudioSegment
11
  from pydub.silence import detect_silence
12
  from scipy.io import wavfile
 
25
  return args
26
 
27
 
28
+ def audio_trim_by_pydub_scipy(filename: str,
29
+ signal: np.ndarray,
30
+ sample_rate: int,
31
+ silence_threshold: int = -40,
32
+ min_silence_len: float = 1000,
33
+ min_kept_silence: float = 200,
34
+ mode: str = "trim",
35
+ ):
36
+ audio = AudioSegment.from_file(filename, format="wav")
37
+ length = len(audio)
38
+
39
+ silent_ranges = detect_silence(audio, min_silence_len=min_silence_len, silence_thresh=silence_threshold)
40
+
41
+ output_dir = Path(tempfile.gettempdir()) / "audio_edit/trim"
42
+ output_dir.mkdir(parents=True, exist_ok=True)
43
+ output_file = output_dir / f"{uuid.uuid4()}.wav"
44
+ output_file = output_file.as_posix()
45
+
46
+ if len(silent_ranges) == 0:
47
+ audio.export(output_file)
48
+ ext = {
49
+ "begin": 0,
50
+ "end": length,
51
+ "origin_length": length,
52
+ }
53
+ return output_file, ext
54
+
55
+ begin_silence = silent_ranges[0]
56
+ begin = 0
57
+ if begin_silence[0] == 0:
58
+ begin = max(0, begin_silence[1] - min_kept_silence)
59
+
60
+ end_silence = silent_ranges[-1]
61
+ end = length
62
+ if end_silence[1] == length:
63
+ end = min(length, end_silence[0] + min_kept_silence)
64
+
65
+ if mode == "trim":
66
+ pass
67
+ elif mode == "rtrim":
68
+ begin = 0
69
+ elif mode == "ltrim":
70
+ end = length
71
+
72
+ begin = int(begin / 1000 * sample_rate)
73
+ end = int(end / 1000 * sample_rate)
74
+
75
+ trimmed_signal = signal[begin:end]
76
+ wavfile.write(
77
+ output_file,
78
+ rate=sample_rate,
79
+ data=trimmed_signal
80
+ )
81
+
82
+ ext = {
83
+ "begin": begin,
84
+ "end": end,
85
+ "origin_length": length,
86
+ }
87
+ return output_file, ext
88
+
89
+
90
  def audio_trim_by_pydub(filename: str,
91
+ signal: np.ndarray,
92
+ sample_rate: int,
93
  silence_threshold: int = -40,
94
  min_silence_len: float = 1000,
95
  min_kept_silence: float = 200,
96
  mode: str = "trim"
97
  ):
98
+ """
99
+ 测试之后发现这个东西将音频质量降低了很多.
100
+ """
101
  audio = AudioSegment.from_file(filename, format="wav")
102
  length = len(audio)
103
 
 
146
 
147
 
148
  def audio_trim_by_librosa(filename: str,
149
+ signal: np.ndarray,
150
+ sample_rate: int,
151
  top_db: float = 60,
152
  frame_length: int = 2048,
153
  hop_length: int = 512,
154
  mode: str = "trim",
155
  **kwargs
156
  ):
157
+ signal_temp, _ = librosa.load(filename, sr=sample_rate, mono=False)
158
  length = len(signal)
159
 
160
  _, index= librosa.effects.trim(
161
+ signal_temp,
162
  top_db=top_db, frame_length=frame_length,
163
  hop_length=hop_length,
164
  **kwargs
 
195
 
196
 
197
  engine_to_function = {
198
+ "pydub_scipy": audio_trim_by_pydub_scipy,
199
  "pydub": audio_trim_by_pydub,
200
  "librosa": audio_trim_by_librosa,
201
  }
202
 
203
 
204
+ def audio_trim(filename: str,
205
+ signal: np.ndarray,
206
+ sample_rate: int,
207
+ engine: str = "librosa", **kwargs):
208
  function = engine_to_function.get(engine)
209
  if function is None:
210
  raise AssertionError(f"invalid engine: {engine}")
211
 
212
+ return function(filename, signal, sample_rate, **kwargs)
213
 
214
 
215
  def main():
toolbox/obs/__init__.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ #!/usr/bin/python3
2
+ # -*- coding: utf-8 -*-
3
+
4
+
5
+ if __name__ == "__main__":
6
+ pass
toolbox/obs/aliyun_oss.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/python3
2
+ # -*- coding: utf-8 -*-
3
+ import json
4
+ import logging
5
+ import time
6
+
7
+ import oss2
8
+ from oss2.credentials import StaticCredentialsProvider
9
+ from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_fixed
10
+
11
+ api_logger = logging.getLogger("api")
12
+
13
+
14
+ class AliyunOSS(object):
15
+ """
16
+ https://help.aliyun.com/zh/oss/developer-reference/getting-started-with-oss-sdk-for-python#0cf90ff8b28eg
17
+ """
18
+ def __init__(self,
19
+ endpoint_url: str,
20
+ region: str,
21
+ secret_key: str,
22
+ secret_id: str,
23
+ bucket: str,
24
+ ):
25
+ self.endpoint_url = endpoint_url
26
+ self.region = region
27
+ self.secret_key = secret_key
28
+ self.secret_id = secret_id
29
+ self.bucket = bucket
30
+
31
+ self.auth = oss2.ProviderAuthV4(StaticCredentialsProvider(
32
+ self.secret_key, self.secret_id
33
+ ))
34
+ self.bucket = oss2.Bucket(self.auth, self.endpoint_url, self.bucket, region=self.region)
35
+
36
+ async def upload_by_filename(self, local_filename: str, cos_filename: str):
37
+ file_in_bytes = open(local_filename, "rb")
38
+ response = await self.upload_by_bytes(file_in_bytes, cos_filename)
39
+ return response
40
+
41
+ @retry(
42
+ wait=wait_fixed(0.5),
43
+ stop=stop_after_attempt(3),
44
+ before_sleep=before_sleep_log(api_logger, logging.ERROR),
45
+ )
46
+ async def upload_by_bytes(self, data_bytes: bytes, url_path: str) -> dict:
47
+
48
+ start_time = time.time()
49
+ try:
50
+ response = self.bucket.put_object(url_path, data_bytes)
51
+
52
+ msg = "success"
53
+ rsp_text = json.dumps({
54
+ "status": response.status,
55
+ "etag": response.etag,
56
+ })
57
+
58
+ time_cost = time.time() - start_time
59
+ api_logger.info(f"s3|{msg}|{time_cost:.3f}s|{self.endpoint_url}|{self.bucket}|{url_path}|{rsp_text}")
60
+ except Exception as e:
61
+ msg = f"{type(e)}: {str(e)}"
62
+ rsp_text = ""
63
+
64
+ time_cost = time.time() - start_time
65
+ api_logger.info(f"s3|{msg}|{time_cost:.3f}s|{self.endpoint_url}|{self.bucket}|{url_path}|{rsp_text}")
66
+ raise e
67
+ return response
68
+
69
+
70
+ if __name__ == "__main__":
71
+ pass
toolbox/obs/obs.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/python3
2
+ # -*- coding: utf-8 -*-
3
+ from toolbox.obs.tencent_cos import TencentCos
4
+ from toolbox.obs.aliyun_oss import AliyunOSS
5
+
6
+
7
+ class OBSManager(object):
8
+ def __init__(self,
9
+ obs_supplier: str,
10
+ url_prefix: str,
11
+ endpoint_url: str,
12
+ region: str,
13
+ secret_id: str,
14
+ secret_key: str,
15
+ bucket: str,
16
+ debug: bool = False
17
+ ):
18
+ self.obs_supplier = obs_supplier
19
+ self.url_prefix = url_prefix
20
+ self.endpoint_url = endpoint_url
21
+ self.region = region
22
+ self.secret_id = secret_id
23
+ self.secret_key = secret_key
24
+ self.bucket = bucket
25
+ self.debug = debug
26
+
27
+ if obs_supplier == "tencent":
28
+ obs_cls = TencentCos
29
+ elif obs_supplier == "aliyun":
30
+ obs_cls = AliyunOSS
31
+ else:
32
+ raise AssertionError
33
+
34
+ self.obs_client = obs_cls(
35
+ endpoint_url=endpoint_url,
36
+ region=region,
37
+ secret_key=secret_key,
38
+ secret_id=secret_id,
39
+ bucket=bucket,
40
+ )
41
+
42
+ async def upload_file_to_obs(self, filename: str, url_path: str):
43
+
44
+ with open(filename, "rb") as f:
45
+ data_bytes = f.read()
46
+
47
+ cos_file_url = f"{self.url_prefix}/{url_path}"
48
+
49
+ if not self.debug:
50
+ result = await self.obs_client.upload_by_bytes(
51
+ data_bytes=data_bytes,
52
+ url_path=url_path,
53
+ )
54
+ return cos_file_url
55
+
56
+
57
+ if __name__ == "__main__":
58
+ pass
toolbox/obs/tencent_cos.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/python3
2
+ # -*- coding: utf-8 -*-
3
+ import json
4
+ import logging
5
+ import time
6
+
7
+ from aiobotocore.session import get_session
8
+ from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_fixed
9
+
10
+ api_logger = logging.getLogger("api")
11
+
12
+
13
+ class TencentCos(object):
14
+ def __init__(self,
15
+ endpoint_url: str,
16
+ region: str,
17
+ secret_key: str,
18
+ secret_id: str,
19
+ bucket: str,
20
+ ):
21
+ self.endpoint_url = endpoint_url
22
+ self.region = region
23
+ self.secret_key = secret_key
24
+ self.secret_id = secret_id
25
+ self.bucket = bucket
26
+
27
+ async def upload_by_filename(self, local_filename: str, cos_filename: str):
28
+ file_in_bytes = open(local_filename, "rb")
29
+ response = await self.upload_by_bytes(file_in_bytes, cos_filename)
30
+ return response
31
+
32
+ @retry(
33
+ wait=wait_fixed(0.5),
34
+ stop=stop_after_attempt(3),
35
+ before_sleep=before_sleep_log(api_logger, logging.ERROR),
36
+ )
37
+ async def upload_by_bytes(self, data_bytes: bytes, url_path: str) -> dict:
38
+
39
+ start_time = time.time()
40
+ try:
41
+ session = get_session()
42
+
43
+ # https://obs.{region}.myhuaweicloud.com
44
+ async with session.create_client(
45
+ "s3",
46
+ endpoint_url=self.endpoint_url,
47
+ region_name=self.region,
48
+ aws_secret_access_key=self.secret_key,
49
+ aws_access_key_id=self.secret_id,
50
+ ) as client:
51
+ response = await client.put_object(
52
+ Bucket=self.bucket,
53
+ Key=url_path,
54
+ Body=data_bytes
55
+ )
56
+
57
+ msg = "success"
58
+ rsp_text = json.dumps(response)
59
+
60
+ time_cost = time.time() - start_time
61
+ api_logger.info(f"s3|{msg}|{time_cost:.3f}s|{self.endpoint_url}|{self.bucket}|{url_path}|{rsp_text}")
62
+ except Exception as e:
63
+ msg = f"{type(e)}: {str(e)}"
64
+ rsp_text = ""
65
+
66
+ time_cost = time.time() - start_time
67
+ api_logger.info(f"s3|{msg}|{time_cost:.3f}s|{self.endpoint_url}|{self.bucket}|{url_path}|{rsp_text}")
68
+ raise e
69
+ return response
70
+
71
+
72
+ if __name__ == "__main__":
73
+ pass