[ update ]: Repair ML Pipeline
Browse files
.github/workflows/gru_pipeline.yaml
CHANGED
@@ -27,38 +27,16 @@ jobs:
|
|
27 |
persist-credentials: false
|
28 |
fetch-depth: 1
|
29 |
|
30 |
-
# - name: Read pipeline schedule date
|
31 |
-
# id: read_schedule
|
32 |
-
# run: |
|
33 |
-
# SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
|
34 |
-
# echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
|
35 |
-
|
36 |
-
# - name: Get current date
|
37 |
-
# id: get_date
|
38 |
-
# run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
39 |
-
|
40 |
-
# - name: Check if dates match
|
41 |
-
# id: date_check
|
42 |
-
# run: |
|
43 |
-
# if [ "$schedule_date" = "$current_date" ]; then
|
44 |
-
# echo "match=true" >> $GITHUB_ENV
|
45 |
-
# else
|
46 |
-
# echo "match=false" >> $GITHUB_ENV
|
47 |
-
# fi
|
48 |
-
|
49 |
- name: Scraping Yahoo Finance
|
50 |
-
# if: env.match != 'true'
|
51 |
run: |
|
52 |
mkdir datasets
|
53 |
go run scraper.go \
|
54 |
-
--symbols-file=./postman/
|
55 |
|
56 |
- name: Zip Datasets
|
57 |
-
# if: env.match != 'true'
|
58 |
run: zip -r datasets.zip datasets
|
59 |
|
60 |
-
- name: Store Datasets to Google Drive
|
61 |
-
# if: env.match != 'true'
|
62 |
uses: adityak74/google-drive-upload-git-action@main
|
63 |
with:
|
64 |
credentials: ${{ secrets.GDRIVE_GRU_CRED }}
|
@@ -68,20 +46,17 @@ jobs:
|
|
68 |
overwrite: "true"
|
69 |
|
70 |
- name: Upload Artifact (datasets)
|
71 |
-
# if: env.match != 'true'
|
72 |
uses: actions/upload-artifact@v3
|
73 |
with:
|
74 |
name: datasets
|
75 |
path: datasets.zip
|
76 |
|
77 |
- name: Remove Temporarary Files and Directories
|
78 |
-
# if: env.match != 'true'
|
79 |
run: |
|
80 |
rm datasets.zip
|
81 |
rm -rf datasets
|
82 |
|
83 |
|
84 |
-
|
85 |
preprocessing_training:
|
86 |
name: Preprocessing and Training
|
87 |
runs-on: ubuntu-latest
|
@@ -98,31 +73,10 @@ jobs:
|
|
98 |
persist-credentials: false
|
99 |
fetch-depth: 1
|
100 |
|
101 |
-
# - name: Read pipeline schedule date
|
102 |
-
# id: read_schedule
|
103 |
-
# run: |
|
104 |
-
# SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
|
105 |
-
# echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
|
106 |
-
|
107 |
-
# - name: Get current date
|
108 |
-
# id: get_date
|
109 |
-
# run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
110 |
-
|
111 |
-
# - name: Check if dates match
|
112 |
-
# id: date_check
|
113 |
-
# run: |
|
114 |
-
# if [ "$schedule_date" = "$current_date" ]; then
|
115 |
-
# echo "match=true" >> $GITHUB_ENV
|
116 |
-
# else
|
117 |
-
# echo "match=false" >> $GITHUB_ENV
|
118 |
-
# fi
|
119 |
-
|
120 |
- name: Install Libraries
|
121 |
-
# if: env.match != 'true'
|
122 |
run: pip install -r requirements.txt
|
123 |
|
124 |
- name: Download Artifact (datasets)
|
125 |
-
# if: env.match != 'true'
|
126 |
uses: actions/download-artifact@v3
|
127 |
with:
|
128 |
name: datasets
|
@@ -130,14 +84,14 @@ jobs:
|
|
130 |
- name: Install Cython and Build ML Module
|
131 |
run: |
|
132 |
apt-get update && \
|
133 |
-
apt-get install -y
|
|
|
134 |
|
135 |
pip install cython
|
136 |
cd training && \
|
137 |
python setup.py build_ext --inplace && cd ..
|
138 |
|
139 |
- name: Modeling and Training
|
140 |
-
# if: env.match != 'true'
|
141 |
run: |
|
142 |
unzip datasets.zip
|
143 |
|
@@ -146,20 +100,18 @@ jobs:
|
|
146 |
mkdir posttrained
|
147 |
|
148 |
python trainingcli.py \
|
149 |
-
--epochs=200
|
150 |
-
--batchs=32
|
151 |
-
--sequences=
|
152 |
--algorithm=GRU
|
153 |
|
154 |
- name: Zip Posttrained, Models, and Pickles
|
155 |
-
# if: env.match != 'true'
|
156 |
run: |
|
157 |
zip -r models.zip models
|
158 |
zip -r pickles.zip pickles
|
159 |
zip -r posttrained.zip posttrained
|
160 |
|
161 |
-
- name: Store Models to Google Drive
|
162 |
-
# if: env.match != 'true'
|
163 |
uses: adityak74/google-drive-upload-git-action@main
|
164 |
with:
|
165 |
credentials: ${{ secrets.GDRIVE_GRU_CRED }}
|
@@ -168,8 +120,7 @@ jobs:
|
|
168 |
name: models.zip
|
169 |
overwrite: "true"
|
170 |
|
171 |
-
- name: Store Pickles to Google Drive
|
172 |
-
# if: env.match != 'true'
|
173 |
uses: adityak74/google-drive-upload-git-action@main
|
174 |
with:
|
175 |
credentials: ${{ secrets.GDRIVE_GRU_CRED }}
|
@@ -178,8 +129,7 @@ jobs:
|
|
178 |
name: pickles.zip
|
179 |
overwrite: "true"
|
180 |
|
181 |
-
- name: Store Posttrained to Google Drive
|
182 |
-
# if: env.match != 'true'
|
183 |
uses: adityak74/google-drive-upload-git-action@main
|
184 |
with:
|
185 |
credentials: ${{ secrets.GDRIVE_GRU_CRED }}
|
@@ -189,7 +139,6 @@ jobs:
|
|
189 |
overwrite: "true"
|
190 |
|
191 |
- name: Remove Temporarary Files and Directories
|
192 |
-
# if: env.match != 'true'
|
193 |
run: |
|
194 |
rm models.zip
|
195 |
rm pickles.zip
|
@@ -202,16 +151,14 @@ jobs:
|
|
202 |
rm -rf posttrained
|
203 |
|
204 |
- name: Flush All Caching Data (Redis)
|
205 |
-
# if: env.match != 'true'
|
206 |
run: |
|
207 |
sudo apt install -y redis-tools
|
208 |
echo "FLUSHALL" | redis-cli --tls \
|
209 |
-
-h ${{ secrets.REDIS_HOST
|
210 |
-
-p ${{ secrets.REDIS_PORT
|
211 |
-a ${{ secrets.REDIS_TOKEN }}
|
212 |
|
213 |
|
214 |
-
|
215 |
tebakaja_crypto_space-0:
|
216 |
name: crypto-forecast-svc-0
|
217 |
runs-on: ubuntu-latest
|
@@ -232,50 +179,29 @@ jobs:
|
|
232 |
persist-credentials: false
|
233 |
fetch-depth: 1000
|
234 |
|
235 |
-
|
236 |
-
|
237 |
-
# run: |
|
238 |
-
# SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
|
239 |
-
# echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
|
240 |
-
|
241 |
-
# - name: Get current date
|
242 |
-
# id: get_date
|
243 |
-
# run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
244 |
-
|
245 |
-
# - name: Check if dates match
|
246 |
-
# id: date_check
|
247 |
-
# run: |
|
248 |
-
# if [ "$schedule_date" = "$current_date" ]; then
|
249 |
-
# echo "match=true" >> $GITHUB_ENV
|
250 |
-
# else
|
251 |
-
# echo "match=false" >> $GITHUB_ENV
|
252 |
-
# fi
|
253 |
|
254 |
- name: Configure git
|
255 |
-
# if: env.match != 'true'
|
256 |
run: |
|
257 |
git config --local user.email "[email protected]"
|
258 |
git config --local user.name "qywok"
|
259 |
|
260 |
- name: Pull changes from remote
|
261 |
-
# if: env.match != 'true'
|
262 |
run: |
|
263 |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
|
264 |
(git merge --strategy-option theirs)
|
265 |
|
266 |
- name: Add and commit changes
|
267 |
-
# if: env.match != 'true'
|
268 |
run: |
|
269 |
git add -A
|
270 |
git diff-index --quiet HEAD || git commit -m "Model Deployment"
|
271 |
|
272 |
- name: Push to Hugging Face
|
273 |
-
# if: env.match != 'true'
|
274 |
run: |
|
275 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
276 |
|
277 |
|
278 |
-
|
279 |
tebakaja_crypto_space-1:
|
280 |
name: crypto-forecast-svc-1
|
281 |
runs-on: ubuntu-latest
|
@@ -296,50 +222,29 @@ jobs:
|
|
296 |
persist-credentials: false
|
297 |
fetch-depth: 1000
|
298 |
|
299 |
-
|
300 |
-
|
301 |
-
# run: |
|
302 |
-
# SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
|
303 |
-
# echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
|
304 |
-
|
305 |
-
# - name: Get current date
|
306 |
-
# id: get_date
|
307 |
-
# run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
308 |
-
|
309 |
-
# - name: Check if dates match
|
310 |
-
# id: date_check
|
311 |
-
# run: |
|
312 |
-
# if [ "$schedule_date" = "$current_date" ]; then
|
313 |
-
# echo "match=true" >> $GITHUB_ENV
|
314 |
-
# else
|
315 |
-
# echo "match=false" >> $GITHUB_ENV
|
316 |
-
# fi
|
317 |
|
318 |
- name: Configure git
|
319 |
-
# if: env.match != 'true'
|
320 |
run: |
|
321 |
git config --local user.email "[email protected]"
|
322 |
git config --local user.name "qywok"
|
323 |
|
324 |
- name: Pull changes from remote
|
325 |
-
# if: env.match != 'true'
|
326 |
run: |
|
327 |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
|
328 |
(git merge --strategy-option theirs)
|
329 |
|
330 |
- name: Add and commit changes
|
331 |
-
# if: env.match != 'true'
|
332 |
run: |
|
333 |
git add -A
|
334 |
git diff-index --quiet HEAD || git commit -m "Model Deployment"
|
335 |
|
336 |
- name: Push to Hugging Face
|
337 |
-
# if: env.match != 'true'
|
338 |
run: |
|
339 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
340 |
|
341 |
|
342 |
-
|
343 |
tebakaja_crypto_space-2:
|
344 |
name: crypto-forecast-svc-2
|
345 |
runs-on: ubuntu-latest
|
@@ -360,50 +265,29 @@ jobs:
|
|
360 |
persist-credentials: false
|
361 |
fetch-depth: 1000
|
362 |
|
363 |
-
|
364 |
-
|
365 |
-
# run: |
|
366 |
-
# SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
|
367 |
-
# echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
|
368 |
-
|
369 |
-
# - name: Get current date
|
370 |
-
# id: get_date
|
371 |
-
# run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
372 |
-
|
373 |
-
# - name: Check if dates match
|
374 |
-
# id: date_check
|
375 |
-
# run: |
|
376 |
-
# if [ "$schedule_date" = "$current_date" ]; then
|
377 |
-
# echo "match=true" >> $GITHUB_ENV
|
378 |
-
# else
|
379 |
-
# echo "match=false" >> $GITHUB_ENV
|
380 |
-
# fi
|
381 |
|
382 |
- name: Configure git
|
383 |
-
# if: env.match != 'true'
|
384 |
run: |
|
385 |
git config --local user.email "[email protected]"
|
386 |
git config --local user.name "qywok"
|
387 |
|
388 |
- name: Pull changes from remote
|
389 |
-
# if: env.match != 'true'
|
390 |
run: |
|
391 |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
|
392 |
(git merge --strategy-option theirs)
|
393 |
|
394 |
- name: Add and commit changes
|
395 |
-
# if: env.match != 'true'
|
396 |
run: |
|
397 |
git add -A
|
398 |
git diff-index --quiet HEAD || git commit -m "Model Deployment"
|
399 |
|
400 |
- name: Push to Hugging Face
|
401 |
-
# if: env.match != 'true'
|
402 |
run: |
|
403 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
404 |
|
405 |
|
406 |
-
|
407 |
tebakaja_crypto_space-3:
|
408 |
name: crypto-forecast-svc-3
|
409 |
runs-on: ubuntu-latest
|
@@ -424,50 +308,29 @@ jobs:
|
|
424 |
persist-credentials: false
|
425 |
fetch-depth: 1000
|
426 |
|
427 |
-
|
428 |
-
|
429 |
-
# run: |
|
430 |
-
# SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
|
431 |
-
# echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
|
432 |
-
|
433 |
-
# - name: Get current date
|
434 |
-
# id: get_date
|
435 |
-
# run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
436 |
-
|
437 |
-
# - name: Check if dates match
|
438 |
-
# id: date_check
|
439 |
-
# run: |
|
440 |
-
# if [ "$schedule_date" = "$current_date" ]; then
|
441 |
-
# echo "match=true" >> $GITHUB_ENV
|
442 |
-
# else
|
443 |
-
# echo "match=false" >> $GITHUB_ENV
|
444 |
-
# fi
|
445 |
|
446 |
- name: Configure git
|
447 |
-
# if: env.match != 'true'
|
448 |
run: |
|
449 |
git config --local user.email "[email protected]"
|
450 |
git config --local user.name "qywok"
|
451 |
|
452 |
- name: Pull changes from remote
|
453 |
-
# if: env.match != 'true'
|
454 |
run: |
|
455 |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
|
456 |
(git merge --strategy-option theirs)
|
457 |
|
458 |
- name: Add and commit changes
|
459 |
-
# if: env.match != 'true'
|
460 |
run: |
|
461 |
git add -A
|
462 |
git diff-index --quiet HEAD || git commit -m "Model Deployment"
|
463 |
|
464 |
- name: Push to Hugging Face
|
465 |
-
# if: env.match != 'true'
|
466 |
run: |
|
467 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
468 |
|
469 |
|
470 |
-
|
471 |
tebakaja_crypto_space-4:
|
472 |
name: crypto-forecast-svc-4
|
473 |
runs-on: ubuntu-latest
|
@@ -488,50 +351,29 @@ jobs:
|
|
488 |
persist-credentials: false
|
489 |
fetch-depth: 1000
|
490 |
|
491 |
-
|
492 |
-
|
493 |
-
# run: |
|
494 |
-
# SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
|
495 |
-
# echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
|
496 |
-
|
497 |
-
# - name: Get current date
|
498 |
-
# id: get_date
|
499 |
-
# run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
500 |
-
|
501 |
-
# - name: Check if dates match
|
502 |
-
# id: date_check
|
503 |
-
# run: |
|
504 |
-
# if [ "$schedule_date" = "$current_date" ]; then
|
505 |
-
# echo "match=true" >> $GITHUB_ENV
|
506 |
-
# else
|
507 |
-
# echo "match=false" >> $GITHUB_ENV
|
508 |
-
# fi
|
509 |
|
510 |
- name: Configure git
|
511 |
-
# if: env.match != 'true'
|
512 |
run: |
|
513 |
git config --local user.email "[email protected]"
|
514 |
git config --local user.name "qywok"
|
515 |
|
516 |
- name: Pull changes from remote
|
517 |
-
# if: env.match != 'true'
|
518 |
run: |
|
519 |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
|
520 |
(git merge --strategy-option theirs)
|
521 |
|
522 |
- name: Add and commit changes
|
523 |
-
# if: env.match != 'true'
|
524 |
run: |
|
525 |
git add -A
|
526 |
git diff-index --quiet HEAD || git commit -m "Model Deployment"
|
527 |
|
528 |
- name: Push to Hugging Face
|
529 |
-
# if: env.match != 'true'
|
530 |
run: |
|
531 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
532 |
|
533 |
|
534 |
-
|
535 |
functional_testing:
|
536 |
name: Functional Testing
|
537 |
runs-on: ubuntu-latest
|
@@ -552,47 +394,8 @@ jobs:
|
|
552 |
persist-credentials: false
|
553 |
fetch-depth: 1
|
554 |
|
555 |
-
# - name: Read pipeline schedule date
|
556 |
-
# id: read_schedule
|
557 |
-
# run: |
|
558 |
-
# SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
|
559 |
-
# echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
|
560 |
-
|
561 |
-
# - name: Get current date
|
562 |
-
# id: get_date
|
563 |
-
# run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
564 |
-
|
565 |
-
# - name: Check if dates match
|
566 |
-
# id: date_check
|
567 |
-
# run: |
|
568 |
-
# if [ "$schedule_date" = "$current_date" ]; then
|
569 |
-
# echo "match=true" >> $GITHUB_ENV
|
570 |
-
# else
|
571 |
-
# echo "match=false" >> $GITHUB_ENV
|
572 |
-
# fi
|
573 |
-
|
574 |
- name: Testing Proxy Endpoints
|
575 |
-
# if: env.match != 'true'
|
576 |
run: |
|
577 |
sleep 60
|
578 |
chmod +x endpoints_test.sh && ./endpoints_test.sh
|
579 |
|
580 |
-
# - name: Set Pipeline Schedule
|
581 |
-
# if: env.match != 'true'
|
582 |
-
# run: echo "$(date +'%Y-%m-%d')" > schedulers/gru_schedule.ctl
|
583 |
-
|
584 |
-
# - name: Commit changes
|
585 |
-
# if: env.match != 'true'
|
586 |
-
# run: |
|
587 |
-
# git config --local user.email "[email protected]"
|
588 |
-
# git config --local user.name "belajarqywok"
|
589 |
-
# git add -A
|
590 |
-
# git commit -m "Data Extraction, Training, and Modeling"
|
591 |
-
|
592 |
-
# - name: Push changes
|
593 |
-
# if: env.match != 'true'
|
594 |
-
# uses: ad-m/github-push-action@master
|
595 |
-
# with:
|
596 |
-
# github_token: ${{ secrets.GH_TOKEN }}
|
597 |
-
# branch: main
|
598 |
-
|
|
|
27 |
persist-credentials: false
|
28 |
fetch-depth: 1
|
29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
30 |
- name: Scraping Yahoo Finance
|
|
|
31 |
run: |
|
32 |
mkdir datasets
|
33 |
go run scraper.go \
|
34 |
+
--symbols-file=./postman/symbols.json
|
35 |
|
36 |
- name: Zip Datasets
|
|
|
37 |
run: zip -r datasets.zip datasets
|
38 |
|
39 |
+
- name: Store Datasets to Google Drive
|
|
|
40 |
uses: adityak74/google-drive-upload-git-action@main
|
41 |
with:
|
42 |
credentials: ${{ secrets.GDRIVE_GRU_CRED }}
|
|
|
46 |
overwrite: "true"
|
47 |
|
48 |
- name: Upload Artifact (datasets)
|
|
|
49 |
uses: actions/upload-artifact@v3
|
50 |
with:
|
51 |
name: datasets
|
52 |
path: datasets.zip
|
53 |
|
54 |
- name: Remove Temporarary Files and Directories
|
|
|
55 |
run: |
|
56 |
rm datasets.zip
|
57 |
rm -rf datasets
|
58 |
|
59 |
|
|
|
60 |
preprocessing_training:
|
61 |
name: Preprocessing and Training
|
62 |
runs-on: ubuntu-latest
|
|
|
73 |
persist-credentials: false
|
74 |
fetch-depth: 1
|
75 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
76 |
- name: Install Libraries
|
|
|
77 |
run: pip install -r requirements.txt
|
78 |
|
79 |
- name: Download Artifact (datasets)
|
|
|
80 |
uses: actions/download-artifact@v3
|
81 |
with:
|
82 |
name: datasets
|
|
|
84 |
- name: Install Cython and Build ML Module
|
85 |
run: |
|
86 |
apt-get update && \
|
87 |
+
apt-get install -y \
|
88 |
+
gcc python3-dev gnupg
|
89 |
|
90 |
pip install cython
|
91 |
cd training && \
|
92 |
python setup.py build_ext --inplace && cd ..
|
93 |
|
94 |
- name: Modeling and Training
|
|
|
95 |
run: |
|
96 |
unzip datasets.zip
|
97 |
|
|
|
100 |
mkdir posttrained
|
101 |
|
102 |
python trainingcli.py \
|
103 |
+
--epochs=200 \
|
104 |
+
--batchs=32 \
|
105 |
+
--sequences=60 \
|
106 |
--algorithm=GRU
|
107 |
|
108 |
- name: Zip Posttrained, Models, and Pickles
|
|
|
109 |
run: |
|
110 |
zip -r models.zip models
|
111 |
zip -r pickles.zip pickles
|
112 |
zip -r posttrained.zip posttrained
|
113 |
|
114 |
+
- name: Store Models to Google Drive
|
|
|
115 |
uses: adityak74/google-drive-upload-git-action@main
|
116 |
with:
|
117 |
credentials: ${{ secrets.GDRIVE_GRU_CRED }}
|
|
|
120 |
name: models.zip
|
121 |
overwrite: "true"
|
122 |
|
123 |
+
- name: Store Pickles to Google Drive
|
|
|
124 |
uses: adityak74/google-drive-upload-git-action@main
|
125 |
with:
|
126 |
credentials: ${{ secrets.GDRIVE_GRU_CRED }}
|
|
|
129 |
name: pickles.zip
|
130 |
overwrite: "true"
|
131 |
|
132 |
+
- name: Store Posttrained to Google Drive
|
|
|
133 |
uses: adityak74/google-drive-upload-git-action@main
|
134 |
with:
|
135 |
credentials: ${{ secrets.GDRIVE_GRU_CRED }}
|
|
|
139 |
overwrite: "true"
|
140 |
|
141 |
- name: Remove Temporarary Files and Directories
|
|
|
142 |
run: |
|
143 |
rm models.zip
|
144 |
rm pickles.zip
|
|
|
151 |
rm -rf posttrained
|
152 |
|
153 |
- name: Flush All Caching Data (Redis)
|
|
|
154 |
run: |
|
155 |
sudo apt install -y redis-tools
|
156 |
echo "FLUSHALL" | redis-cli --tls \
|
157 |
+
-h ${{ secrets.REDIS_HOST }} \
|
158 |
+
-p ${{ secrets.REDIS_PORT }} \
|
159 |
-a ${{ secrets.REDIS_TOKEN }}
|
160 |
|
161 |
|
|
|
162 |
tebakaja_crypto_space-0:
|
163 |
name: crypto-forecast-svc-0
|
164 |
runs-on: ubuntu-latest
|
|
|
179 |
persist-credentials: false
|
180 |
fetch-depth: 1000
|
181 |
|
182 |
+
- name: Check git status
|
183 |
+
run: git status
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
184 |
|
185 |
- name: Configure git
|
|
|
186 |
run: |
|
187 |
git config --local user.email "[email protected]"
|
188 |
git config --local user.name "qywok"
|
189 |
|
190 |
- name: Pull changes from remote
|
|
|
191 |
run: |
|
192 |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
|
193 |
(git merge --strategy-option theirs)
|
194 |
|
195 |
- name: Add and commit changes
|
|
|
196 |
run: |
|
197 |
git add -A
|
198 |
git diff-index --quiet HEAD || git commit -m "Model Deployment"
|
199 |
|
200 |
- name: Push to Hugging Face
|
|
|
201 |
run: |
|
202 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
203 |
|
204 |
|
|
|
205 |
tebakaja_crypto_space-1:
|
206 |
name: crypto-forecast-svc-1
|
207 |
runs-on: ubuntu-latest
|
|
|
222 |
persist-credentials: false
|
223 |
fetch-depth: 1000
|
224 |
|
225 |
+
- name: Check git status
|
226 |
+
run: git status
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
227 |
|
228 |
- name: Configure git
|
|
|
229 |
run: |
|
230 |
git config --local user.email "[email protected]"
|
231 |
git config --local user.name "qywok"
|
232 |
|
233 |
- name: Pull changes from remote
|
|
|
234 |
run: |
|
235 |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
|
236 |
(git merge --strategy-option theirs)
|
237 |
|
238 |
- name: Add and commit changes
|
|
|
239 |
run: |
|
240 |
git add -A
|
241 |
git diff-index --quiet HEAD || git commit -m "Model Deployment"
|
242 |
|
243 |
- name: Push to Hugging Face
|
|
|
244 |
run: |
|
245 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
246 |
|
247 |
|
|
|
248 |
tebakaja_crypto_space-2:
|
249 |
name: crypto-forecast-svc-2
|
250 |
runs-on: ubuntu-latest
|
|
|
265 |
persist-credentials: false
|
266 |
fetch-depth: 1000
|
267 |
|
268 |
+
- name: Check git status
|
269 |
+
run: git status
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
270 |
|
271 |
- name: Configure git
|
|
|
272 |
run: |
|
273 |
git config --local user.email "[email protected]"
|
274 |
git config --local user.name "qywok"
|
275 |
|
276 |
- name: Pull changes from remote
|
|
|
277 |
run: |
|
278 |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
|
279 |
(git merge --strategy-option theirs)
|
280 |
|
281 |
- name: Add and commit changes
|
|
|
282 |
run: |
|
283 |
git add -A
|
284 |
git diff-index --quiet HEAD || git commit -m "Model Deployment"
|
285 |
|
286 |
- name: Push to Hugging Face
|
|
|
287 |
run: |
|
288 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
289 |
|
290 |
|
|
|
291 |
tebakaja_crypto_space-3:
|
292 |
name: crypto-forecast-svc-3
|
293 |
runs-on: ubuntu-latest
|
|
|
308 |
persist-credentials: false
|
309 |
fetch-depth: 1000
|
310 |
|
311 |
+
- name: Check git status
|
312 |
+
run: git status
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
313 |
|
314 |
- name: Configure git
|
|
|
315 |
run: |
|
316 |
git config --local user.email "[email protected]"
|
317 |
git config --local user.name "qywok"
|
318 |
|
319 |
- name: Pull changes from remote
|
|
|
320 |
run: |
|
321 |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
|
322 |
(git merge --strategy-option theirs)
|
323 |
|
324 |
- name: Add and commit changes
|
|
|
325 |
run: |
|
326 |
git add -A
|
327 |
git diff-index --quiet HEAD || git commit -m "Model Deployment"
|
328 |
|
329 |
- name: Push to Hugging Face
|
|
|
330 |
run: |
|
331 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
332 |
|
333 |
|
|
|
334 |
tebakaja_crypto_space-4:
|
335 |
name: crypto-forecast-svc-4
|
336 |
runs-on: ubuntu-latest
|
|
|
351 |
persist-credentials: false
|
352 |
fetch-depth: 1000
|
353 |
|
354 |
+
- name: Check git status
|
355 |
+
run: git status
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
356 |
|
357 |
- name: Configure git
|
|
|
358 |
run: |
|
359 |
git config --local user.email "[email protected]"
|
360 |
git config --local user.name "qywok"
|
361 |
|
362 |
- name: Pull changes from remote
|
|
|
363 |
run: |
|
364 |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
|
365 |
(git merge --strategy-option theirs)
|
366 |
|
367 |
- name: Add and commit changes
|
|
|
368 |
run: |
|
369 |
git add -A
|
370 |
git diff-index --quiet HEAD || git commit -m "Model Deployment"
|
371 |
|
372 |
- name: Push to Hugging Face
|
|
|
373 |
run: |
|
374 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
375 |
|
376 |
|
|
|
377 |
functional_testing:
|
378 |
name: Functional Testing
|
379 |
runs-on: ubuntu-latest
|
|
|
394 |
persist-credentials: false
|
395 |
fetch-depth: 1
|
396 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
397 |
- name: Testing Proxy Endpoints
|
|
|
398 |
run: |
|
399 |
sleep 60
|
400 |
chmod +x endpoints_test.sh && ./endpoints_test.sh
|
401 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.github/workflows/lstm_gru_pipeline.yaml
CHANGED
@@ -27,38 +27,16 @@ jobs:
|
|
27 |
persist-credentials: false
|
28 |
fetch-depth: 1
|
29 |
|
30 |
-
# - name: Read pipeline schedule date
|
31 |
-
# id: read_schedule
|
32 |
-
# run: |
|
33 |
-
# SCHEDULE_DATE=$(cat schedulers/lstm_gru_schedule.ctl)
|
34 |
-
# echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
|
35 |
-
|
36 |
-
# - name: Get current date
|
37 |
-
# id: get_date
|
38 |
-
# run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
39 |
-
|
40 |
-
# - name: Check if dates match
|
41 |
-
# id: date_check
|
42 |
-
# run: |
|
43 |
-
# if [ "$schedule_date" = "$current_date" ]; then
|
44 |
-
# echo "match=true" >> $GITHUB_ENV
|
45 |
-
# else
|
46 |
-
# echo "match=false" >> $GITHUB_ENV
|
47 |
-
# fi
|
48 |
-
|
49 |
- name: Scraping Yahoo Finance
|
50 |
-
# if: env.match != 'true'
|
51 |
run: |
|
52 |
mkdir datasets
|
53 |
go run scraper.go \
|
54 |
-
--symbols-file=./postman/
|
55 |
|
56 |
- name: Zip Datasets
|
57 |
-
# if: env.match != 'true'
|
58 |
run: zip -r datasets.zip datasets
|
59 |
|
60 |
-
- name: Store Datasets to Google Drive
|
61 |
-
# if: env.match != 'true'
|
62 |
uses: adityak74/google-drive-upload-git-action@main
|
63 |
with:
|
64 |
credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
|
@@ -68,20 +46,17 @@ jobs:
|
|
68 |
overwrite: "true"
|
69 |
|
70 |
- name: Upload Artifact (datasets)
|
71 |
-
# if: env.match != 'true'
|
72 |
uses: actions/upload-artifact@v3
|
73 |
with:
|
74 |
name: datasets
|
75 |
path: datasets.zip
|
76 |
|
77 |
- name: Remove Temporarary Files and Directories
|
78 |
-
# if: env.match != 'true'
|
79 |
run: |
|
80 |
rm datasets.zip
|
81 |
rm -rf datasets
|
82 |
|
83 |
|
84 |
-
|
85 |
preprocessing_training:
|
86 |
name: Preprocessing and Training
|
87 |
runs-on: ubuntu-latest
|
@@ -98,31 +73,10 @@ jobs:
|
|
98 |
persist-credentials: false
|
99 |
fetch-depth: 1
|
100 |
|
101 |
-
# - name: Read pipeline schedule date
|
102 |
-
# id: read_schedule
|
103 |
-
# run: |
|
104 |
-
# SCHEDULE_DATE=$(cat schedulers/lstm_gru_schedule.ctl)
|
105 |
-
# echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
|
106 |
-
|
107 |
-
# - name: Get current date
|
108 |
-
# id: get_date
|
109 |
-
# run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
110 |
-
|
111 |
-
# - name: Check if dates match
|
112 |
-
# id: date_check
|
113 |
-
# run: |
|
114 |
-
# if [ "$schedule_date" = "$current_date" ]; then
|
115 |
-
# echo "match=true" >> $GITHUB_ENV
|
116 |
-
# else
|
117 |
-
# echo "match=false" >> $GITHUB_ENV
|
118 |
-
# fi
|
119 |
-
|
120 |
- name: Install Libraries
|
121 |
-
# if: env.match != 'true'
|
122 |
run: pip install -r requirements.txt
|
123 |
|
124 |
- name: Download Artifact (datasets)
|
125 |
-
# if: env.match != 'true'
|
126 |
uses: actions/download-artifact@v3
|
127 |
with:
|
128 |
name: datasets
|
@@ -130,14 +84,14 @@ jobs:
|
|
130 |
- name: Install Cython and Build ML Module
|
131 |
run: |
|
132 |
apt-get update && \
|
133 |
-
apt-get install -y
|
|
|
134 |
|
135 |
pip install cython
|
136 |
cd training && \
|
137 |
python setup.py build_ext --inplace && cd ..
|
138 |
|
139 |
- name: Modeling and Training
|
140 |
-
# if: env.match != 'true'
|
141 |
run: |
|
142 |
unzip datasets.zip
|
143 |
|
@@ -146,24 +100,18 @@ jobs:
|
|
146 |
mkdir posttrained
|
147 |
|
148 |
python trainingcli.py \
|
149 |
-
--epochs=200
|
150 |
-
--batchs=32
|
151 |
-
--sequences=
|
152 |
--algorithm=LSTM_GRU
|
153 |
-
|
154 |
-
# - name: Set Pipeline Schedule
|
155 |
-
# if: env.match != 'true'
|
156 |
-
# run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_gru_schedule.ctl
|
157 |
|
158 |
- name: Zip Posttrained, Models, and Pickles
|
159 |
-
# if: env.match != 'true'
|
160 |
run: |
|
161 |
zip -r models.zip models
|
162 |
zip -r pickles.zip pickles
|
163 |
zip -r posttrained.zip posttrained
|
164 |
|
165 |
-
- name: Store Models to Google Drive
|
166 |
-
# if: env.match != 'true'
|
167 |
uses: adityak74/google-drive-upload-git-action@main
|
168 |
with:
|
169 |
credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
|
@@ -172,8 +120,7 @@ jobs:
|
|
172 |
name: models.zip
|
173 |
overwrite: "true"
|
174 |
|
175 |
-
- name: Store Pickles to Google Drive
|
176 |
-
# if: env.match != 'true'
|
177 |
uses: adityak74/google-drive-upload-git-action@main
|
178 |
with:
|
179 |
credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
|
@@ -182,8 +129,7 @@ jobs:
|
|
182 |
name: pickles.zip
|
183 |
overwrite: "true"
|
184 |
|
185 |
-
- name: Store Posttrained to Google Drive
|
186 |
-
# if: env.match != 'true'
|
187 |
uses: adityak74/google-drive-upload-git-action@main
|
188 |
with:
|
189 |
credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
|
@@ -193,7 +139,6 @@ jobs:
|
|
193 |
overwrite: "true"
|
194 |
|
195 |
- name: Remove Temporarary Files and Directories
|
196 |
-
# if: env.match != 'true'
|
197 |
run: |
|
198 |
rm models.zip
|
199 |
rm pickles.zip
|
@@ -205,22 +150,6 @@ jobs:
|
|
205 |
rm -rf datasets
|
206 |
rm -rf posttrained
|
207 |
|
208 |
-
# - name: Commit changes
|
209 |
-
# if: env.match != 'true'
|
210 |
-
# run: |
|
211 |
-
# git config --local user.email "[email protected]"
|
212 |
-
# git config --local user.name "belajarqywok"
|
213 |
-
# git add -A
|
214 |
-
# git commit -m "Data Extraction, Training, and Modeling"
|
215 |
-
|
216 |
-
# - name: Push changes
|
217 |
-
# if: env.match != 'true'
|
218 |
-
# uses: ad-m/github-push-action@master
|
219 |
-
# with:
|
220 |
-
# github_token: ${{ secrets.GH_TOKEN }}
|
221 |
-
# branch: main
|
222 |
-
|
223 |
-
|
224 |
|
225 |
tebakaja_crypto_space-0:
|
226 |
name: crypto-forecast-svc-0
|
@@ -265,7 +194,6 @@ jobs:
|
|
265 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
266 |
|
267 |
|
268 |
-
|
269 |
tebakaja_crypto_space-1:
|
270 |
name: crypto-forecast-svc-1
|
271 |
runs-on: ubuntu-latest
|
@@ -309,7 +237,6 @@ jobs:
|
|
309 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
310 |
|
311 |
|
312 |
-
|
313 |
tebakaja_crypto_space-2:
|
314 |
name: crypto-forecast-svc-2
|
315 |
runs-on: ubuntu-latest
|
@@ -353,7 +280,6 @@ jobs:
|
|
353 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
354 |
|
355 |
|
356 |
-
|
357 |
tebakaja_crypto_space-3:
|
358 |
name: crypto-forecast-svc-3
|
359 |
runs-on: ubuntu-latest
|
@@ -397,7 +323,6 @@ jobs:
|
|
397 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
398 |
|
399 |
|
400 |
-
|
401 |
tebakaja_crypto_space-4:
|
402 |
name: crypto-forecast-svc-4
|
403 |
runs-on: ubuntu-latest
|
@@ -441,7 +366,6 @@ jobs:
|
|
441 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
442 |
|
443 |
|
444 |
-
|
445 |
functional_testing:
|
446 |
name: Functional Testing
|
447 |
runs-on: ubuntu-latest
|
|
|
27 |
persist-credentials: false
|
28 |
fetch-depth: 1
|
29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
30 |
- name: Scraping Yahoo Finance
|
|
|
31 |
run: |
|
32 |
mkdir datasets
|
33 |
go run scraper.go \
|
34 |
+
--symbols-file=./postman/symbols.json
|
35 |
|
36 |
- name: Zip Datasets
|
|
|
37 |
run: zip -r datasets.zip datasets
|
38 |
|
39 |
+
- name: Store Datasets to Google Drive
|
|
|
40 |
uses: adityak74/google-drive-upload-git-action@main
|
41 |
with:
|
42 |
credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
|
|
|
46 |
overwrite: "true"
|
47 |
|
48 |
- name: Upload Artifact (datasets)
|
|
|
49 |
uses: actions/upload-artifact@v3
|
50 |
with:
|
51 |
name: datasets
|
52 |
path: datasets.zip
|
53 |
|
54 |
- name: Remove Temporarary Files and Directories
|
|
|
55 |
run: |
|
56 |
rm datasets.zip
|
57 |
rm -rf datasets
|
58 |
|
59 |
|
|
|
60 |
preprocessing_training:
|
61 |
name: Preprocessing and Training
|
62 |
runs-on: ubuntu-latest
|
|
|
73 |
persist-credentials: false
|
74 |
fetch-depth: 1
|
75 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
76 |
- name: Install Libraries
|
|
|
77 |
run: pip install -r requirements.txt
|
78 |
|
79 |
- name: Download Artifact (datasets)
|
|
|
80 |
uses: actions/download-artifact@v3
|
81 |
with:
|
82 |
name: datasets
|
|
|
84 |
- name: Install Cython and Build ML Module
|
85 |
run: |
|
86 |
apt-get update && \
|
87 |
+
apt-get install -y \
|
88 |
+
gcc python3-dev gnupg
|
89 |
|
90 |
pip install cython
|
91 |
cd training && \
|
92 |
python setup.py build_ext --inplace && cd ..
|
93 |
|
94 |
- name: Modeling and Training
|
|
|
95 |
run: |
|
96 |
unzip datasets.zip
|
97 |
|
|
|
100 |
mkdir posttrained
|
101 |
|
102 |
python trainingcli.py \
|
103 |
+
--epochs=200 \
|
104 |
+
--batchs=32 \
|
105 |
+
--sequences=60 \
|
106 |
--algorithm=LSTM_GRU
|
|
|
|
|
|
|
|
|
107 |
|
108 |
- name: Zip Posttrained, Models, and Pickles
|
|
|
109 |
run: |
|
110 |
zip -r models.zip models
|
111 |
zip -r pickles.zip pickles
|
112 |
zip -r posttrained.zip posttrained
|
113 |
|
114 |
+
- name: Store Models to Google Drive
|
|
|
115 |
uses: adityak74/google-drive-upload-git-action@main
|
116 |
with:
|
117 |
credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
|
|
|
120 |
name: models.zip
|
121 |
overwrite: "true"
|
122 |
|
123 |
+
- name: Store Pickles to Google Drive
|
|
|
124 |
uses: adityak74/google-drive-upload-git-action@main
|
125 |
with:
|
126 |
credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
|
|
|
129 |
name: pickles.zip
|
130 |
overwrite: "true"
|
131 |
|
132 |
+
- name: Store Posttrained to Google Drive
|
|
|
133 |
uses: adityak74/google-drive-upload-git-action@main
|
134 |
with:
|
135 |
credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
|
|
|
139 |
overwrite: "true"
|
140 |
|
141 |
- name: Remove Temporarary Files and Directories
|
|
|
142 |
run: |
|
143 |
rm models.zip
|
144 |
rm pickles.zip
|
|
|
150 |
rm -rf datasets
|
151 |
rm -rf posttrained
|
152 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
153 |
|
154 |
tebakaja_crypto_space-0:
|
155 |
name: crypto-forecast-svc-0
|
|
|
194 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
195 |
|
196 |
|
|
|
197 |
tebakaja_crypto_space-1:
|
198 |
name: crypto-forecast-svc-1
|
199 |
runs-on: ubuntu-latest
|
|
|
237 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
238 |
|
239 |
|
|
|
240 |
tebakaja_crypto_space-2:
|
241 |
name: crypto-forecast-svc-2
|
242 |
runs-on: ubuntu-latest
|
|
|
280 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
281 |
|
282 |
|
|
|
283 |
tebakaja_crypto_space-3:
|
284 |
name: crypto-forecast-svc-3
|
285 |
runs-on: ubuntu-latest
|
|
|
323 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
324 |
|
325 |
|
|
|
326 |
tebakaja_crypto_space-4:
|
327 |
name: crypto-forecast-svc-4
|
328 |
runs-on: ubuntu-latest
|
|
|
366 |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
|
367 |
|
368 |
|
|
|
369 |
functional_testing:
|
370 |
name: Functional Testing
|
371 |
runs-on: ubuntu-latest
|
.github/workflows/lstm_pipeline.yaml
CHANGED
@@ -27,38 +27,16 @@ jobs:
|
|
27 |
persist-credentials: false
|
28 |
fetch-depth: 1
|
29 |
|
30 |
-
# - name: Read pipeline schedule date
|
31 |
-
# id: read_schedule
|
32 |
-
# run: |
|
33 |
-
# SCHEDULE_DATE=$(cat schedulers/lstm_schedule.ctl)
|
34 |
-
# echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
|
35 |
-
|
36 |
-
# - name: Get current date
|
37 |
-
# id: get_date
|
38 |
-
# run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
39 |
-
|
40 |
-
# - name: Check if dates match
|
41 |
-
# id: date_check
|
42 |
-
# run: |
|
43 |
-
# if [ "$schedule_date" = "$current_date" ]; then
|
44 |
-
# echo "match=true" >> $GITHUB_ENV
|
45 |
-
# else
|
46 |
-
# echo "match=false" >> $GITHUB_ENV
|
47 |
-
# fi
|
48 |
-
|
49 |
- name: Scraping Yahoo Finance
|
50 |
-
# if: env.match != 'true'
|
51 |
run: |
|
52 |
mkdir datasets
|
53 |
go run scraper.go \
|
54 |
-
--symbols-file=./postman/
|
55 |
|
56 |
- name: Zip Datasets
|
57 |
-
# if: env.match != 'true'
|
58 |
run: zip -r datasets.zip datasets
|
59 |
|
60 |
-
- name: Store Datasets to Google Drive
|
61 |
-
# if: env.match != 'true'
|
62 |
uses: adityak74/google-drive-upload-git-action@main
|
63 |
with:
|
64 |
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
|
@@ -68,14 +46,12 @@ jobs:
|
|
68 |
overwrite: "true"
|
69 |
|
70 |
- name: Upload Artifact (datasets)
|
71 |
-
# if: env.match != 'true'
|
72 |
uses: actions/upload-artifact@v3
|
73 |
with:
|
74 |
name: datasets
|
75 |
path: datasets.zip
|
76 |
|
77 |
- name: Remove Temporarary Files and Directories
|
78 |
-
# if: env.match != 'true'
|
79 |
run: |
|
80 |
rm datasets.zip
|
81 |
rm -rf datasets
|
@@ -97,31 +73,10 @@ jobs:
|
|
97 |
persist-credentials: false
|
98 |
fetch-depth: 1
|
99 |
|
100 |
-
# - name: Read pipeline schedule date
|
101 |
-
# id: read_schedule
|
102 |
-
# run: |
|
103 |
-
# SCHEDULE_DATE=$(cat schedulers/lstm_schedule.ctl)
|
104 |
-
# echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
|
105 |
-
|
106 |
-
# - name: Get current date
|
107 |
-
# id: get_date
|
108 |
-
# run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
109 |
-
|
110 |
-
# - name: Check if dates match
|
111 |
-
# id: date_check
|
112 |
-
# run: |
|
113 |
-
# if [ "$schedule_date" = "$current_date" ]; then
|
114 |
-
# echo "match=true" >> $GITHUB_ENV
|
115 |
-
# else
|
116 |
-
# echo "match=false" >> $GITHUB_ENV
|
117 |
-
# fi
|
118 |
-
|
119 |
- name: Install Libraries
|
120 |
-
# if: env.match != 'true'
|
121 |
run: pip install -r requirements.txt
|
122 |
|
123 |
- name: Download Artifact (datasets)
|
124 |
-
# if: env.match != 'true'
|
125 |
uses: actions/download-artifact@v3
|
126 |
with:
|
127 |
name: datasets
|
@@ -136,7 +91,6 @@ jobs:
|
|
136 |
python setup.py build_ext --inplace && cd ..
|
137 |
|
138 |
- name: Modeling and Training
|
139 |
-
# if: env.match != 'true'
|
140 |
run: |
|
141 |
unzip datasets.zip
|
142 |
|
@@ -145,24 +99,18 @@ jobs:
|
|
145 |
mkdir posttrained
|
146 |
|
147 |
python trainingcli.py \
|
148 |
-
--epochs=200
|
149 |
-
--batchs=32
|
150 |
-
--sequences=
|
151 |
--algorithm=LSTM
|
152 |
-
|
153 |
-
# - name: Set Pipeline Schedule
|
154 |
-
# if: env.match != 'true'
|
155 |
-
# run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_schedule.ctl
|
156 |
|
157 |
- name: Zip Posttrained, Models, and Pickles
|
158 |
-
# if: env.match != 'true'
|
159 |
run: |
|
160 |
zip -r models.zip models
|
161 |
zip -r pickles.zip pickles
|
162 |
zip -r posttrained.zip posttrained
|
163 |
|
164 |
-
- name: Store Models to Google Drive
|
165 |
-
# if: env.match != 'true'
|
166 |
uses: adityak74/google-drive-upload-git-action@main
|
167 |
with:
|
168 |
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
|
@@ -171,8 +119,7 @@ jobs:
|
|
171 |
name: models.zip
|
172 |
overwrite: "true"
|
173 |
|
174 |
-
- name: Store Pickles to Google Drive
|
175 |
-
# if: env.match != 'true'
|
176 |
uses: adityak74/google-drive-upload-git-action@main
|
177 |
with:
|
178 |
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
|
@@ -181,8 +128,7 @@ jobs:
|
|
181 |
name: pickles.zip
|
182 |
overwrite: "true"
|
183 |
|
184 |
-
- name: Store Posttrained to Google Drive
|
185 |
-
# if: env.match != 'true'
|
186 |
uses: adityak74/google-drive-upload-git-action@main
|
187 |
with:
|
188 |
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
|
@@ -192,7 +138,6 @@ jobs:
|
|
192 |
overwrite: "true"
|
193 |
|
194 |
- name: Remove Temporarary Files and Directories
|
195 |
-
# if: env.match != 'true'
|
196 |
run: |
|
197 |
rm models.zip
|
198 |
rm pickles.zip
|
@@ -204,21 +149,6 @@ jobs:
|
|
204 |
rm -rf datasets
|
205 |
rm -rf posttrained
|
206 |
|
207 |
-
# - name: Commit changes
|
208 |
-
# if: env.match != 'true'
|
209 |
-
# run: |
|
210 |
-
# git config --local user.email "[email protected]"
|
211 |
-
# git config --local user.name "belajarqywok"
|
212 |
-
# git add -A
|
213 |
-
# git commit -m "Data Extraction, Training, and Modeling"
|
214 |
-
|
215 |
-
# - name: Push changes
|
216 |
-
# if: env.match != 'true'
|
217 |
-
# uses: ad-m/github-push-action@master
|
218 |
-
# with:
|
219 |
-
# github_token: ${{ secrets.GH_TOKEN }}
|
220 |
-
# branch: main
|
221 |
-
|
222 |
|
223 |
tebakaja_crypto_space-0:
|
224 |
name: crypto-forecast-svc-0
|
|
|
27 |
persist-credentials: false
|
28 |
fetch-depth: 1
|
29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
30 |
- name: Scraping Yahoo Finance
|
|
|
31 |
run: |
|
32 |
mkdir datasets
|
33 |
go run scraper.go \
|
34 |
+
--symbols-file=./postman/symbols.json
|
35 |
|
36 |
- name: Zip Datasets
|
|
|
37 |
run: zip -r datasets.zip datasets
|
38 |
|
39 |
+
- name: Store Datasets to Google Drive
|
|
|
40 |
uses: adityak74/google-drive-upload-git-action@main
|
41 |
with:
|
42 |
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
|
|
|
46 |
overwrite: "true"
|
47 |
|
48 |
- name: Upload Artifact (datasets)
|
|
|
49 |
uses: actions/upload-artifact@v3
|
50 |
with:
|
51 |
name: datasets
|
52 |
path: datasets.zip
|
53 |
|
54 |
- name: Remove Temporarary Files and Directories
|
|
|
55 |
run: |
|
56 |
rm datasets.zip
|
57 |
rm -rf datasets
|
|
|
73 |
persist-credentials: false
|
74 |
fetch-depth: 1
|
75 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
76 |
- name: Install Libraries
|
|
|
77 |
run: pip install -r requirements.txt
|
78 |
|
79 |
- name: Download Artifact (datasets)
|
|
|
80 |
uses: actions/download-artifact@v3
|
81 |
with:
|
82 |
name: datasets
|
|
|
91 |
python setup.py build_ext --inplace && cd ..
|
92 |
|
93 |
- name: Modeling and Training
|
|
|
94 |
run: |
|
95 |
unzip datasets.zip
|
96 |
|
|
|
99 |
mkdir posttrained
|
100 |
|
101 |
python trainingcli.py \
|
102 |
+
--epochs=200 \
|
103 |
+
--batchs=32 \
|
104 |
+
--sequences=60 \
|
105 |
--algorithm=LSTM
|
|
|
|
|
|
|
|
|
106 |
|
107 |
- name: Zip Posttrained, Models, and Pickles
|
|
|
108 |
run: |
|
109 |
zip -r models.zip models
|
110 |
zip -r pickles.zip pickles
|
111 |
zip -r posttrained.zip posttrained
|
112 |
|
113 |
+
- name: Store Models to Google Drive
|
|
|
114 |
uses: adityak74/google-drive-upload-git-action@main
|
115 |
with:
|
116 |
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
|
|
|
119 |
name: models.zip
|
120 |
overwrite: "true"
|
121 |
|
122 |
+
- name: Store Pickles to Google Drive
|
|
|
123 |
uses: adityak74/google-drive-upload-git-action@main
|
124 |
with:
|
125 |
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
|
|
|
128 |
name: pickles.zip
|
129 |
overwrite: "true"
|
130 |
|
131 |
+
- name: Store Posttrained to Google Drive
|
|
|
132 |
uses: adityak74/google-drive-upload-git-action@main
|
133 |
with:
|
134 |
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
|
|
|
138 |
overwrite: "true"
|
139 |
|
140 |
- name: Remove Temporarary Files and Directories
|
|
|
141 |
run: |
|
142 |
rm models.zip
|
143 |
rm pickles.zip
|
|
|
149 |
rm -rf datasets
|
150 |
rm -rf posttrained
|
151 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
152 |
|
153 |
tebakaja_crypto_space-0:
|
154 |
name: crypto-forecast-svc-0
|