tebakaja commited on
Commit
995815e
1 Parent(s): 8c63991

[ update ]: Repair ML Pipeline

Browse files
.github/workflows/gru_pipeline.yaml CHANGED
@@ -27,38 +27,16 @@ jobs:
27
  persist-credentials: false
28
  fetch-depth: 1
29
 
30
- # - name: Read pipeline schedule date
31
- # id: read_schedule
32
- # run: |
33
- # SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
34
- # echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
35
-
36
- # - name: Get current date
37
- # id: get_date
38
- # run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
39
-
40
- # - name: Check if dates match
41
- # id: date_check
42
- # run: |
43
- # if [ "$schedule_date" = "$current_date" ]; then
44
- # echo "match=true" >> $GITHUB_ENV
45
- # else
46
- # echo "match=false" >> $GITHUB_ENV
47
- # fi
48
-
49
  - name: Scraping Yahoo Finance
50
- # if: env.match != 'true'
51
  run: |
52
  mkdir datasets
53
  go run scraper.go \
54
- --symbols-file=./postman/symbols_test.json
55
 
56
  - name: Zip Datasets
57
- # if: env.match != 'true'
58
  run: zip -r datasets.zip datasets
59
 
60
- - name: Store Datasets to Google Drive
61
- # if: env.match != 'true'
62
  uses: adityak74/google-drive-upload-git-action@main
63
  with:
64
  credentials: ${{ secrets.GDRIVE_GRU_CRED }}
@@ -68,20 +46,17 @@ jobs:
68
  overwrite: "true"
69
 
70
  - name: Upload Artifact (datasets)
71
- # if: env.match != 'true'
72
  uses: actions/upload-artifact@v3
73
  with:
74
  name: datasets
75
  path: datasets.zip
76
 
77
  - name: Remove Temporarary Files and Directories
78
- # if: env.match != 'true'
79
  run: |
80
  rm datasets.zip
81
  rm -rf datasets
82
 
83
 
84
-
85
  preprocessing_training:
86
  name: Preprocessing and Training
87
  runs-on: ubuntu-latest
@@ -98,31 +73,10 @@ jobs:
98
  persist-credentials: false
99
  fetch-depth: 1
100
 
101
- # - name: Read pipeline schedule date
102
- # id: read_schedule
103
- # run: |
104
- # SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
105
- # echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
106
-
107
- # - name: Get current date
108
- # id: get_date
109
- # run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
110
-
111
- # - name: Check if dates match
112
- # id: date_check
113
- # run: |
114
- # if [ "$schedule_date" = "$current_date" ]; then
115
- # echo "match=true" >> $GITHUB_ENV
116
- # else
117
- # echo "match=false" >> $GITHUB_ENV
118
- # fi
119
-
120
  - name: Install Libraries
121
- # if: env.match != 'true'
122
  run: pip install -r requirements.txt
123
 
124
  - name: Download Artifact (datasets)
125
- # if: env.match != 'true'
126
  uses: actions/download-artifact@v3
127
  with:
128
  name: datasets
@@ -130,14 +84,14 @@ jobs:
130
  - name: Install Cython and Build ML Module
131
  run: |
132
  apt-get update && \
133
- apt-get install -y gcc python3-dev gnupg
 
134
 
135
  pip install cython
136
  cd training && \
137
  python setup.py build_ext --inplace && cd ..
138
 
139
  - name: Modeling and Training
140
- # if: env.match != 'true'
141
  run: |
142
  unzip datasets.zip
143
 
@@ -146,20 +100,18 @@ jobs:
146
  mkdir posttrained
147
 
148
  python trainingcli.py \
149
- --epochs=200 \
150
- --batchs=32 \
151
- --sequences=5 \
152
  --algorithm=GRU
153
 
154
  - name: Zip Posttrained, Models, and Pickles
155
- # if: env.match != 'true'
156
  run: |
157
  zip -r models.zip models
158
  zip -r pickles.zip pickles
159
  zip -r posttrained.zip posttrained
160
 
161
- - name: Store Models to Google Drive
162
- # if: env.match != 'true'
163
  uses: adityak74/google-drive-upload-git-action@main
164
  with:
165
  credentials: ${{ secrets.GDRIVE_GRU_CRED }}
@@ -168,8 +120,7 @@ jobs:
168
  name: models.zip
169
  overwrite: "true"
170
 
171
- - name: Store Pickles to Google Drive
172
- # if: env.match != 'true'
173
  uses: adityak74/google-drive-upload-git-action@main
174
  with:
175
  credentials: ${{ secrets.GDRIVE_GRU_CRED }}
@@ -178,8 +129,7 @@ jobs:
178
  name: pickles.zip
179
  overwrite: "true"
180
 
181
- - name: Store Posttrained to Google Drive
182
- # if: env.match != 'true'
183
  uses: adityak74/google-drive-upload-git-action@main
184
  with:
185
  credentials: ${{ secrets.GDRIVE_GRU_CRED }}
@@ -189,7 +139,6 @@ jobs:
189
  overwrite: "true"
190
 
191
  - name: Remove Temporarary Files and Directories
192
- # if: env.match != 'true'
193
  run: |
194
  rm models.zip
195
  rm pickles.zip
@@ -202,16 +151,14 @@ jobs:
202
  rm -rf posttrained
203
 
204
  - name: Flush All Caching Data (Redis)
205
- # if: env.match != 'true'
206
  run: |
207
  sudo apt install -y redis-tools
208
  echo "FLUSHALL" | redis-cli --tls \
209
- -h ${{ secrets.REDIS_HOST }} \
210
- -p ${{ secrets.REDIS_PORT }} \
211
  -a ${{ secrets.REDIS_TOKEN }}
212
 
213
 
214
-
215
  tebakaja_crypto_space-0:
216
  name: crypto-forecast-svc-0
217
  runs-on: ubuntu-latest
@@ -232,50 +179,29 @@ jobs:
232
  persist-credentials: false
233
  fetch-depth: 1000
234
 
235
- # - name: Read pipeline schedule date
236
- # id: read_schedule
237
- # run: |
238
- # SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
239
- # echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
240
-
241
- # - name: Get current date
242
- # id: get_date
243
- # run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
244
-
245
- # - name: Check if dates match
246
- # id: date_check
247
- # run: |
248
- # if [ "$schedule_date" = "$current_date" ]; then
249
- # echo "match=true" >> $GITHUB_ENV
250
- # else
251
- # echo "match=false" >> $GITHUB_ENV
252
- # fi
253
 
254
  - name: Configure git
255
- # if: env.match != 'true'
256
  run: |
257
  git config --local user.email "[email protected]"
258
  git config --local user.name "qywok"
259
 
260
  - name: Pull changes from remote
261
- # if: env.match != 'true'
262
  run: |
263
  git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
264
  (git merge --strategy-option theirs)
265
 
266
  - name: Add and commit changes
267
- # if: env.match != 'true'
268
  run: |
269
  git add -A
270
  git diff-index --quiet HEAD || git commit -m "Model Deployment"
271
 
272
  - name: Push to Hugging Face
273
- # if: env.match != 'true'
274
  run: |
275
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
276
 
277
 
278
-
279
  tebakaja_crypto_space-1:
280
  name: crypto-forecast-svc-1
281
  runs-on: ubuntu-latest
@@ -296,50 +222,29 @@ jobs:
296
  persist-credentials: false
297
  fetch-depth: 1000
298
 
299
- # - name: Read pipeline schedule date
300
- # id: read_schedule
301
- # run: |
302
- # SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
303
- # echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
304
-
305
- # - name: Get current date
306
- # id: get_date
307
- # run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
308
-
309
- # - name: Check if dates match
310
- # id: date_check
311
- # run: |
312
- # if [ "$schedule_date" = "$current_date" ]; then
313
- # echo "match=true" >> $GITHUB_ENV
314
- # else
315
- # echo "match=false" >> $GITHUB_ENV
316
- # fi
317
 
318
  - name: Configure git
319
- # if: env.match != 'true'
320
  run: |
321
  git config --local user.email "[email protected]"
322
  git config --local user.name "qywok"
323
 
324
  - name: Pull changes from remote
325
- # if: env.match != 'true'
326
  run: |
327
  git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
328
  (git merge --strategy-option theirs)
329
 
330
  - name: Add and commit changes
331
- # if: env.match != 'true'
332
  run: |
333
  git add -A
334
  git diff-index --quiet HEAD || git commit -m "Model Deployment"
335
 
336
  - name: Push to Hugging Face
337
- # if: env.match != 'true'
338
  run: |
339
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
340
 
341
 
342
-
343
  tebakaja_crypto_space-2:
344
  name: crypto-forecast-svc-2
345
  runs-on: ubuntu-latest
@@ -360,50 +265,29 @@ jobs:
360
  persist-credentials: false
361
  fetch-depth: 1000
362
 
363
- # - name: Read pipeline schedule date
364
- # id: read_schedule
365
- # run: |
366
- # SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
367
- # echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
368
-
369
- # - name: Get current date
370
- # id: get_date
371
- # run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
372
-
373
- # - name: Check if dates match
374
- # id: date_check
375
- # run: |
376
- # if [ "$schedule_date" = "$current_date" ]; then
377
- # echo "match=true" >> $GITHUB_ENV
378
- # else
379
- # echo "match=false" >> $GITHUB_ENV
380
- # fi
381
 
382
  - name: Configure git
383
- # if: env.match != 'true'
384
  run: |
385
  git config --local user.email "[email protected]"
386
  git config --local user.name "qywok"
387
 
388
  - name: Pull changes from remote
389
- # if: env.match != 'true'
390
  run: |
391
  git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
392
  (git merge --strategy-option theirs)
393
 
394
  - name: Add and commit changes
395
- # if: env.match != 'true'
396
  run: |
397
  git add -A
398
  git diff-index --quiet HEAD || git commit -m "Model Deployment"
399
 
400
  - name: Push to Hugging Face
401
- # if: env.match != 'true'
402
  run: |
403
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
404
 
405
 
406
-
407
  tebakaja_crypto_space-3:
408
  name: crypto-forecast-svc-3
409
  runs-on: ubuntu-latest
@@ -424,50 +308,29 @@ jobs:
424
  persist-credentials: false
425
  fetch-depth: 1000
426
 
427
- # - name: Read pipeline schedule date
428
- # id: read_schedule
429
- # run: |
430
- # SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
431
- # echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
432
-
433
- # - name: Get current date
434
- # id: get_date
435
- # run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
436
-
437
- # - name: Check if dates match
438
- # id: date_check
439
- # run: |
440
- # if [ "$schedule_date" = "$current_date" ]; then
441
- # echo "match=true" >> $GITHUB_ENV
442
- # else
443
- # echo "match=false" >> $GITHUB_ENV
444
- # fi
445
 
446
  - name: Configure git
447
- # if: env.match != 'true'
448
  run: |
449
  git config --local user.email "[email protected]"
450
  git config --local user.name "qywok"
451
 
452
  - name: Pull changes from remote
453
- # if: env.match != 'true'
454
  run: |
455
  git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
456
  (git merge --strategy-option theirs)
457
 
458
  - name: Add and commit changes
459
- # if: env.match != 'true'
460
  run: |
461
  git add -A
462
  git diff-index --quiet HEAD || git commit -m "Model Deployment"
463
 
464
  - name: Push to Hugging Face
465
- # if: env.match != 'true'
466
  run: |
467
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
468
 
469
 
470
-
471
  tebakaja_crypto_space-4:
472
  name: crypto-forecast-svc-4
473
  runs-on: ubuntu-latest
@@ -488,50 +351,29 @@ jobs:
488
  persist-credentials: false
489
  fetch-depth: 1000
490
 
491
- # - name: Read pipeline schedule date
492
- # id: read_schedule
493
- # run: |
494
- # SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
495
- # echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
496
-
497
- # - name: Get current date
498
- # id: get_date
499
- # run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
500
-
501
- # - name: Check if dates match
502
- # id: date_check
503
- # run: |
504
- # if [ "$schedule_date" = "$current_date" ]; then
505
- # echo "match=true" >> $GITHUB_ENV
506
- # else
507
- # echo "match=false" >> $GITHUB_ENV
508
- # fi
509
 
510
  - name: Configure git
511
- # if: env.match != 'true'
512
  run: |
513
  git config --local user.email "[email protected]"
514
  git config --local user.name "qywok"
515
 
516
  - name: Pull changes from remote
517
- # if: env.match != 'true'
518
  run: |
519
  git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
520
  (git merge --strategy-option theirs)
521
 
522
  - name: Add and commit changes
523
- # if: env.match != 'true'
524
  run: |
525
  git add -A
526
  git diff-index --quiet HEAD || git commit -m "Model Deployment"
527
 
528
  - name: Push to Hugging Face
529
- # if: env.match != 'true'
530
  run: |
531
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
532
 
533
 
534
-
535
  functional_testing:
536
  name: Functional Testing
537
  runs-on: ubuntu-latest
@@ -552,47 +394,8 @@ jobs:
552
  persist-credentials: false
553
  fetch-depth: 1
554
 
555
- # - name: Read pipeline schedule date
556
- # id: read_schedule
557
- # run: |
558
- # SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl)
559
- # echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
560
-
561
- # - name: Get current date
562
- # id: get_date
563
- # run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
564
-
565
- # - name: Check if dates match
566
- # id: date_check
567
- # run: |
568
- # if [ "$schedule_date" = "$current_date" ]; then
569
- # echo "match=true" >> $GITHUB_ENV
570
- # else
571
- # echo "match=false" >> $GITHUB_ENV
572
- # fi
573
-
574
  - name: Testing Proxy Endpoints
575
- # if: env.match != 'true'
576
  run: |
577
  sleep 60
578
  chmod +x endpoints_test.sh && ./endpoints_test.sh
579
 
580
- # - name: Set Pipeline Schedule
581
- # if: env.match != 'true'
582
- # run: echo "$(date +'%Y-%m-%d')" > schedulers/gru_schedule.ctl
583
-
584
- # - name: Commit changes
585
- # if: env.match != 'true'
586
- # run: |
587
- # git config --local user.email "[email protected]"
588
- # git config --local user.name "belajarqywok"
589
- # git add -A
590
- # git commit -m "Data Extraction, Training, and Modeling"
591
-
592
- # - name: Push changes
593
- # if: env.match != 'true'
594
- # uses: ad-m/github-push-action@master
595
- # with:
596
- # github_token: ${{ secrets.GH_TOKEN }}
597
- # branch: main
598
-
 
27
  persist-credentials: false
28
  fetch-depth: 1
29
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  - name: Scraping Yahoo Finance
 
31
  run: |
32
  mkdir datasets
33
  go run scraper.go \
34
+ --symbols-file=./postman/symbols.json
35
 
36
  - name: Zip Datasets
 
37
  run: zip -r datasets.zip datasets
38
 
39
+ - name: Store Datasets to Google Drive
 
40
  uses: adityak74/google-drive-upload-git-action@main
41
  with:
42
  credentials: ${{ secrets.GDRIVE_GRU_CRED }}
 
46
  overwrite: "true"
47
 
48
  - name: Upload Artifact (datasets)
 
49
  uses: actions/upload-artifact@v3
50
  with:
51
  name: datasets
52
  path: datasets.zip
53
 
54
  - name: Remove Temporarary Files and Directories
 
55
  run: |
56
  rm datasets.zip
57
  rm -rf datasets
58
 
59
 
 
60
  preprocessing_training:
61
  name: Preprocessing and Training
62
  runs-on: ubuntu-latest
 
73
  persist-credentials: false
74
  fetch-depth: 1
75
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
76
  - name: Install Libraries
 
77
  run: pip install -r requirements.txt
78
 
79
  - name: Download Artifact (datasets)
 
80
  uses: actions/download-artifact@v3
81
  with:
82
  name: datasets
 
84
  - name: Install Cython and Build ML Module
85
  run: |
86
  apt-get update && \
87
+ apt-get install -y \
88
+ gcc python3-dev gnupg
89
 
90
  pip install cython
91
  cd training && \
92
  python setup.py build_ext --inplace && cd ..
93
 
94
  - name: Modeling and Training
 
95
  run: |
96
  unzip datasets.zip
97
 
 
100
  mkdir posttrained
101
 
102
  python trainingcli.py \
103
+ --epochs=200 \
104
+ --batchs=32 \
105
+ --sequences=60 \
106
  --algorithm=GRU
107
 
108
  - name: Zip Posttrained, Models, and Pickles
 
109
  run: |
110
  zip -r models.zip models
111
  zip -r pickles.zip pickles
112
  zip -r posttrained.zip posttrained
113
 
114
+ - name: Store Models to Google Drive
 
115
  uses: adityak74/google-drive-upload-git-action@main
116
  with:
117
  credentials: ${{ secrets.GDRIVE_GRU_CRED }}
 
120
  name: models.zip
121
  overwrite: "true"
122
 
123
+ - name: Store Pickles to Google Drive
 
124
  uses: adityak74/google-drive-upload-git-action@main
125
  with:
126
  credentials: ${{ secrets.GDRIVE_GRU_CRED }}
 
129
  name: pickles.zip
130
  overwrite: "true"
131
 
132
+ - name: Store Posttrained to Google Drive
 
133
  uses: adityak74/google-drive-upload-git-action@main
134
  with:
135
  credentials: ${{ secrets.GDRIVE_GRU_CRED }}
 
139
  overwrite: "true"
140
 
141
  - name: Remove Temporarary Files and Directories
 
142
  run: |
143
  rm models.zip
144
  rm pickles.zip
 
151
  rm -rf posttrained
152
 
153
  - name: Flush All Caching Data (Redis)
 
154
  run: |
155
  sudo apt install -y redis-tools
156
  echo "FLUSHALL" | redis-cli --tls \
157
+ -h ${{ secrets.REDIS_HOST }} \
158
+ -p ${{ secrets.REDIS_PORT }} \
159
  -a ${{ secrets.REDIS_TOKEN }}
160
 
161
 
 
162
  tebakaja_crypto_space-0:
163
  name: crypto-forecast-svc-0
164
  runs-on: ubuntu-latest
 
179
  persist-credentials: false
180
  fetch-depth: 1000
181
 
182
+ - name: Check git status
183
+ run: git status
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
184
 
185
  - name: Configure git
 
186
  run: |
187
  git config --local user.email "[email protected]"
188
  git config --local user.name "qywok"
189
 
190
  - name: Pull changes from remote
 
191
  run: |
192
  git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
193
  (git merge --strategy-option theirs)
194
 
195
  - name: Add and commit changes
 
196
  run: |
197
  git add -A
198
  git diff-index --quiet HEAD || git commit -m "Model Deployment"
199
 
200
  - name: Push to Hugging Face
 
201
  run: |
202
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
203
 
204
 
 
205
  tebakaja_crypto_space-1:
206
  name: crypto-forecast-svc-1
207
  runs-on: ubuntu-latest
 
222
  persist-credentials: false
223
  fetch-depth: 1000
224
 
225
+ - name: Check git status
226
+ run: git status
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
227
 
228
  - name: Configure git
 
229
  run: |
230
  git config --local user.email "[email protected]"
231
  git config --local user.name "qywok"
232
 
233
  - name: Pull changes from remote
 
234
  run: |
235
  git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
236
  (git merge --strategy-option theirs)
237
 
238
  - name: Add and commit changes
 
239
  run: |
240
  git add -A
241
  git diff-index --quiet HEAD || git commit -m "Model Deployment"
242
 
243
  - name: Push to Hugging Face
 
244
  run: |
245
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
246
 
247
 
 
248
  tebakaja_crypto_space-2:
249
  name: crypto-forecast-svc-2
250
  runs-on: ubuntu-latest
 
265
  persist-credentials: false
266
  fetch-depth: 1000
267
 
268
+ - name: Check git status
269
+ run: git status
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
270
 
271
  - name: Configure git
 
272
  run: |
273
  git config --local user.email "[email protected]"
274
  git config --local user.name "qywok"
275
 
276
  - name: Pull changes from remote
 
277
  run: |
278
  git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
279
  (git merge --strategy-option theirs)
280
 
281
  - name: Add and commit changes
 
282
  run: |
283
  git add -A
284
  git diff-index --quiet HEAD || git commit -m "Model Deployment"
285
 
286
  - name: Push to Hugging Face
 
287
  run: |
288
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
289
 
290
 
 
291
  tebakaja_crypto_space-3:
292
  name: crypto-forecast-svc-3
293
  runs-on: ubuntu-latest
 
308
  persist-credentials: false
309
  fetch-depth: 1000
310
 
311
+ - name: Check git status
312
+ run: git status
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
313
 
314
  - name: Configure git
 
315
  run: |
316
  git config --local user.email "[email protected]"
317
  git config --local user.name "qywok"
318
 
319
  - name: Pull changes from remote
 
320
  run: |
321
  git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
322
  (git merge --strategy-option theirs)
323
 
324
  - name: Add and commit changes
 
325
  run: |
326
  git add -A
327
  git diff-index --quiet HEAD || git commit -m "Model Deployment"
328
 
329
  - name: Push to Hugging Face
 
330
  run: |
331
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
332
 
333
 
 
334
  tebakaja_crypto_space-4:
335
  name: crypto-forecast-svc-4
336
  runs-on: ubuntu-latest
 
351
  persist-credentials: false
352
  fetch-depth: 1000
353
 
354
+ - name: Check git status
355
+ run: git status
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
356
 
357
  - name: Configure git
 
358
  run: |
359
  git config --local user.email "[email protected]"
360
  git config --local user.name "qywok"
361
 
362
  - name: Pull changes from remote
 
363
  run: |
364
  git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
365
  (git merge --strategy-option theirs)
366
 
367
  - name: Add and commit changes
 
368
  run: |
369
  git add -A
370
  git diff-index --quiet HEAD || git commit -m "Model Deployment"
371
 
372
  - name: Push to Hugging Face
 
373
  run: |
374
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
375
 
376
 
 
377
  functional_testing:
378
  name: Functional Testing
379
  runs-on: ubuntu-latest
 
394
  persist-credentials: false
395
  fetch-depth: 1
396
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
397
  - name: Testing Proxy Endpoints
 
398
  run: |
399
  sleep 60
400
  chmod +x endpoints_test.sh && ./endpoints_test.sh
401
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.github/workflows/lstm_gru_pipeline.yaml CHANGED
@@ -27,38 +27,16 @@ jobs:
27
  persist-credentials: false
28
  fetch-depth: 1
29
 
30
- # - name: Read pipeline schedule date
31
- # id: read_schedule
32
- # run: |
33
- # SCHEDULE_DATE=$(cat schedulers/lstm_gru_schedule.ctl)
34
- # echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
35
-
36
- # - name: Get current date
37
- # id: get_date
38
- # run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
39
-
40
- # - name: Check if dates match
41
- # id: date_check
42
- # run: |
43
- # if [ "$schedule_date" = "$current_date" ]; then
44
- # echo "match=true" >> $GITHUB_ENV
45
- # else
46
- # echo "match=false" >> $GITHUB_ENV
47
- # fi
48
-
49
  - name: Scraping Yahoo Finance
50
- # if: env.match != 'true'
51
  run: |
52
  mkdir datasets
53
  go run scraper.go \
54
- --symbols-file=./postman/symbols_test.json
55
 
56
  - name: Zip Datasets
57
- # if: env.match != 'true'
58
  run: zip -r datasets.zip datasets
59
 
60
- - name: Store Datasets to Google Drive
61
- # if: env.match != 'true'
62
  uses: adityak74/google-drive-upload-git-action@main
63
  with:
64
  credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
@@ -68,20 +46,17 @@ jobs:
68
  overwrite: "true"
69
 
70
  - name: Upload Artifact (datasets)
71
- # if: env.match != 'true'
72
  uses: actions/upload-artifact@v3
73
  with:
74
  name: datasets
75
  path: datasets.zip
76
 
77
  - name: Remove Temporarary Files and Directories
78
- # if: env.match != 'true'
79
  run: |
80
  rm datasets.zip
81
  rm -rf datasets
82
 
83
 
84
-
85
  preprocessing_training:
86
  name: Preprocessing and Training
87
  runs-on: ubuntu-latest
@@ -98,31 +73,10 @@ jobs:
98
  persist-credentials: false
99
  fetch-depth: 1
100
 
101
- # - name: Read pipeline schedule date
102
- # id: read_schedule
103
- # run: |
104
- # SCHEDULE_DATE=$(cat schedulers/lstm_gru_schedule.ctl)
105
- # echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
106
-
107
- # - name: Get current date
108
- # id: get_date
109
- # run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
110
-
111
- # - name: Check if dates match
112
- # id: date_check
113
- # run: |
114
- # if [ "$schedule_date" = "$current_date" ]; then
115
- # echo "match=true" >> $GITHUB_ENV
116
- # else
117
- # echo "match=false" >> $GITHUB_ENV
118
- # fi
119
-
120
  - name: Install Libraries
121
- # if: env.match != 'true'
122
  run: pip install -r requirements.txt
123
 
124
  - name: Download Artifact (datasets)
125
- # if: env.match != 'true'
126
  uses: actions/download-artifact@v3
127
  with:
128
  name: datasets
@@ -130,14 +84,14 @@ jobs:
130
  - name: Install Cython and Build ML Module
131
  run: |
132
  apt-get update && \
133
- apt-get install -y gcc python3-dev gnupg
 
134
 
135
  pip install cython
136
  cd training && \
137
  python setup.py build_ext --inplace && cd ..
138
 
139
  - name: Modeling and Training
140
- # if: env.match != 'true'
141
  run: |
142
  unzip datasets.zip
143
 
@@ -146,24 +100,18 @@ jobs:
146
  mkdir posttrained
147
 
148
  python trainingcli.py \
149
- --epochs=200 \
150
- --batchs=32 \
151
- --sequences=5 \
152
  --algorithm=LSTM_GRU
153
-
154
- # - name: Set Pipeline Schedule
155
- # if: env.match != 'true'
156
- # run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_gru_schedule.ctl
157
 
158
  - name: Zip Posttrained, Models, and Pickles
159
- # if: env.match != 'true'
160
  run: |
161
  zip -r models.zip models
162
  zip -r pickles.zip pickles
163
  zip -r posttrained.zip posttrained
164
 
165
- - name: Store Models to Google Drive
166
- # if: env.match != 'true'
167
  uses: adityak74/google-drive-upload-git-action@main
168
  with:
169
  credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
@@ -172,8 +120,7 @@ jobs:
172
  name: models.zip
173
  overwrite: "true"
174
 
175
- - name: Store Pickles to Google Drive
176
- # if: env.match != 'true'
177
  uses: adityak74/google-drive-upload-git-action@main
178
  with:
179
  credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
@@ -182,8 +129,7 @@ jobs:
182
  name: pickles.zip
183
  overwrite: "true"
184
 
185
- - name: Store Posttrained to Google Drive
186
- # if: env.match != 'true'
187
  uses: adityak74/google-drive-upload-git-action@main
188
  with:
189
  credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
@@ -193,7 +139,6 @@ jobs:
193
  overwrite: "true"
194
 
195
  - name: Remove Temporarary Files and Directories
196
- # if: env.match != 'true'
197
  run: |
198
  rm models.zip
199
  rm pickles.zip
@@ -205,22 +150,6 @@ jobs:
205
  rm -rf datasets
206
  rm -rf posttrained
207
 
208
- # - name: Commit changes
209
- # if: env.match != 'true'
210
- # run: |
211
- # git config --local user.email "[email protected]"
212
- # git config --local user.name "belajarqywok"
213
- # git add -A
214
- # git commit -m "Data Extraction, Training, and Modeling"
215
-
216
- # - name: Push changes
217
- # if: env.match != 'true'
218
- # uses: ad-m/github-push-action@master
219
- # with:
220
- # github_token: ${{ secrets.GH_TOKEN }}
221
- # branch: main
222
-
223
-
224
 
225
  tebakaja_crypto_space-0:
226
  name: crypto-forecast-svc-0
@@ -265,7 +194,6 @@ jobs:
265
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
266
 
267
 
268
-
269
  tebakaja_crypto_space-1:
270
  name: crypto-forecast-svc-1
271
  runs-on: ubuntu-latest
@@ -309,7 +237,6 @@ jobs:
309
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
310
 
311
 
312
-
313
  tebakaja_crypto_space-2:
314
  name: crypto-forecast-svc-2
315
  runs-on: ubuntu-latest
@@ -353,7 +280,6 @@ jobs:
353
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
354
 
355
 
356
-
357
  tebakaja_crypto_space-3:
358
  name: crypto-forecast-svc-3
359
  runs-on: ubuntu-latest
@@ -397,7 +323,6 @@ jobs:
397
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
398
 
399
 
400
-
401
  tebakaja_crypto_space-4:
402
  name: crypto-forecast-svc-4
403
  runs-on: ubuntu-latest
@@ -441,7 +366,6 @@ jobs:
441
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
442
 
443
 
444
-
445
  functional_testing:
446
  name: Functional Testing
447
  runs-on: ubuntu-latest
 
27
  persist-credentials: false
28
  fetch-depth: 1
29
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  - name: Scraping Yahoo Finance
 
31
  run: |
32
  mkdir datasets
33
  go run scraper.go \
34
+ --symbols-file=./postman/symbols.json
35
 
36
  - name: Zip Datasets
 
37
  run: zip -r datasets.zip datasets
38
 
39
+ - name: Store Datasets to Google Drive
 
40
  uses: adityak74/google-drive-upload-git-action@main
41
  with:
42
  credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
 
46
  overwrite: "true"
47
 
48
  - name: Upload Artifact (datasets)
 
49
  uses: actions/upload-artifact@v3
50
  with:
51
  name: datasets
52
  path: datasets.zip
53
 
54
  - name: Remove Temporarary Files and Directories
 
55
  run: |
56
  rm datasets.zip
57
  rm -rf datasets
58
 
59
 
 
60
  preprocessing_training:
61
  name: Preprocessing and Training
62
  runs-on: ubuntu-latest
 
73
  persist-credentials: false
74
  fetch-depth: 1
75
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
76
  - name: Install Libraries
 
77
  run: pip install -r requirements.txt
78
 
79
  - name: Download Artifact (datasets)
 
80
  uses: actions/download-artifact@v3
81
  with:
82
  name: datasets
 
84
  - name: Install Cython and Build ML Module
85
  run: |
86
  apt-get update && \
87
+ apt-get install -y \
88
+ gcc python3-dev gnupg
89
 
90
  pip install cython
91
  cd training && \
92
  python setup.py build_ext --inplace && cd ..
93
 
94
  - name: Modeling and Training
 
95
  run: |
96
  unzip datasets.zip
97
 
 
100
  mkdir posttrained
101
 
102
  python trainingcli.py \
103
+ --epochs=200 \
104
+ --batchs=32 \
105
+ --sequences=60 \
106
  --algorithm=LSTM_GRU
 
 
 
 
107
 
108
  - name: Zip Posttrained, Models, and Pickles
 
109
  run: |
110
  zip -r models.zip models
111
  zip -r pickles.zip pickles
112
  zip -r posttrained.zip posttrained
113
 
114
+ - name: Store Models to Google Drive
 
115
  uses: adityak74/google-drive-upload-git-action@main
116
  with:
117
  credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
 
120
  name: models.zip
121
  overwrite: "true"
122
 
123
+ - name: Store Pickles to Google Drive
 
124
  uses: adityak74/google-drive-upload-git-action@main
125
  with:
126
  credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
 
129
  name: pickles.zip
130
  overwrite: "true"
131
 
132
+ - name: Store Posttrained to Google Drive
 
133
  uses: adityak74/google-drive-upload-git-action@main
134
  with:
135
  credentials: ${{ secrets.GDRIVE_LSTM_GRU_CRED }}
 
139
  overwrite: "true"
140
 
141
  - name: Remove Temporarary Files and Directories
 
142
  run: |
143
  rm models.zip
144
  rm pickles.zip
 
150
  rm -rf datasets
151
  rm -rf posttrained
152
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
153
 
154
  tebakaja_crypto_space-0:
155
  name: crypto-forecast-svc-0
 
194
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
195
 
196
 
 
197
  tebakaja_crypto_space-1:
198
  name: crypto-forecast-svc-1
199
  runs-on: ubuntu-latest
 
237
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
238
 
239
 
 
240
  tebakaja_crypto_space-2:
241
  name: crypto-forecast-svc-2
242
  runs-on: ubuntu-latest
 
280
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
281
 
282
 
 
283
  tebakaja_crypto_space-3:
284
  name: crypto-forecast-svc-3
285
  runs-on: ubuntu-latest
 
323
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
324
 
325
 
 
326
  tebakaja_crypto_space-4:
327
  name: crypto-forecast-svc-4
328
  runs-on: ubuntu-latest
 
366
  git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
367
 
368
 
 
369
  functional_testing:
370
  name: Functional Testing
371
  runs-on: ubuntu-latest
.github/workflows/lstm_pipeline.yaml CHANGED
@@ -27,38 +27,16 @@ jobs:
27
  persist-credentials: false
28
  fetch-depth: 1
29
 
30
- # - name: Read pipeline schedule date
31
- # id: read_schedule
32
- # run: |
33
- # SCHEDULE_DATE=$(cat schedulers/lstm_schedule.ctl)
34
- # echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
35
-
36
- # - name: Get current date
37
- # id: get_date
38
- # run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
39
-
40
- # - name: Check if dates match
41
- # id: date_check
42
- # run: |
43
- # if [ "$schedule_date" = "$current_date" ]; then
44
- # echo "match=true" >> $GITHUB_ENV
45
- # else
46
- # echo "match=false" >> $GITHUB_ENV
47
- # fi
48
-
49
  - name: Scraping Yahoo Finance
50
- # if: env.match != 'true'
51
  run: |
52
  mkdir datasets
53
  go run scraper.go \
54
- --symbols-file=./postman/symbols_test.json
55
 
56
  - name: Zip Datasets
57
- # if: env.match != 'true'
58
  run: zip -r datasets.zip datasets
59
 
60
- - name: Store Datasets to Google Drive
61
- # if: env.match != 'true'
62
  uses: adityak74/google-drive-upload-git-action@main
63
  with:
64
  credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
@@ -68,14 +46,12 @@ jobs:
68
  overwrite: "true"
69
 
70
  - name: Upload Artifact (datasets)
71
- # if: env.match != 'true'
72
  uses: actions/upload-artifact@v3
73
  with:
74
  name: datasets
75
  path: datasets.zip
76
 
77
  - name: Remove Temporarary Files and Directories
78
- # if: env.match != 'true'
79
  run: |
80
  rm datasets.zip
81
  rm -rf datasets
@@ -97,31 +73,10 @@ jobs:
97
  persist-credentials: false
98
  fetch-depth: 1
99
 
100
- # - name: Read pipeline schedule date
101
- # id: read_schedule
102
- # run: |
103
- # SCHEDULE_DATE=$(cat schedulers/lstm_schedule.ctl)
104
- # echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
105
-
106
- # - name: Get current date
107
- # id: get_date
108
- # run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
109
-
110
- # - name: Check if dates match
111
- # id: date_check
112
- # run: |
113
- # if [ "$schedule_date" = "$current_date" ]; then
114
- # echo "match=true" >> $GITHUB_ENV
115
- # else
116
- # echo "match=false" >> $GITHUB_ENV
117
- # fi
118
-
119
  - name: Install Libraries
120
- # if: env.match != 'true'
121
  run: pip install -r requirements.txt
122
 
123
  - name: Download Artifact (datasets)
124
- # if: env.match != 'true'
125
  uses: actions/download-artifact@v3
126
  with:
127
  name: datasets
@@ -136,7 +91,6 @@ jobs:
136
  python setup.py build_ext --inplace && cd ..
137
 
138
  - name: Modeling and Training
139
- # if: env.match != 'true'
140
  run: |
141
  unzip datasets.zip
142
 
@@ -145,24 +99,18 @@ jobs:
145
  mkdir posttrained
146
 
147
  python trainingcli.py \
148
- --epochs=200 \
149
- --batchs=32 \
150
- --sequences=5 \
151
  --algorithm=LSTM
152
-
153
- # - name: Set Pipeline Schedule
154
- # if: env.match != 'true'
155
- # run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_schedule.ctl
156
 
157
  - name: Zip Posttrained, Models, and Pickles
158
- # if: env.match != 'true'
159
  run: |
160
  zip -r models.zip models
161
  zip -r pickles.zip pickles
162
  zip -r posttrained.zip posttrained
163
 
164
- - name: Store Models to Google Drive
165
- # if: env.match != 'true'
166
  uses: adityak74/google-drive-upload-git-action@main
167
  with:
168
  credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
@@ -171,8 +119,7 @@ jobs:
171
  name: models.zip
172
  overwrite: "true"
173
 
174
- - name: Store Pickles to Google Drive
175
- # if: env.match != 'true'
176
  uses: adityak74/google-drive-upload-git-action@main
177
  with:
178
  credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
@@ -181,8 +128,7 @@ jobs:
181
  name: pickles.zip
182
  overwrite: "true"
183
 
184
- - name: Store Posttrained to Google Drive
185
- # if: env.match != 'true'
186
  uses: adityak74/google-drive-upload-git-action@main
187
  with:
188
  credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
@@ -192,7 +138,6 @@ jobs:
192
  overwrite: "true"
193
 
194
  - name: Remove Temporarary Files and Directories
195
- # if: env.match != 'true'
196
  run: |
197
  rm models.zip
198
  rm pickles.zip
@@ -204,21 +149,6 @@ jobs:
204
  rm -rf datasets
205
  rm -rf posttrained
206
 
207
- # - name: Commit changes
208
- # if: env.match != 'true'
209
- # run: |
210
- # git config --local user.email "[email protected]"
211
- # git config --local user.name "belajarqywok"
212
- # git add -A
213
- # git commit -m "Data Extraction, Training, and Modeling"
214
-
215
- # - name: Push changes
216
- # if: env.match != 'true'
217
- # uses: ad-m/github-push-action@master
218
- # with:
219
- # github_token: ${{ secrets.GH_TOKEN }}
220
- # branch: main
221
-
222
 
223
  tebakaja_crypto_space-0:
224
  name: crypto-forecast-svc-0
 
27
  persist-credentials: false
28
  fetch-depth: 1
29
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  - name: Scraping Yahoo Finance
 
31
  run: |
32
  mkdir datasets
33
  go run scraper.go \
34
+ --symbols-file=./postman/symbols.json
35
 
36
  - name: Zip Datasets
 
37
  run: zip -r datasets.zip datasets
38
 
39
+ - name: Store Datasets to Google Drive
 
40
  uses: adityak74/google-drive-upload-git-action@main
41
  with:
42
  credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
 
46
  overwrite: "true"
47
 
48
  - name: Upload Artifact (datasets)
 
49
  uses: actions/upload-artifact@v3
50
  with:
51
  name: datasets
52
  path: datasets.zip
53
 
54
  - name: Remove Temporarary Files and Directories
 
55
  run: |
56
  rm datasets.zip
57
  rm -rf datasets
 
73
  persist-credentials: false
74
  fetch-depth: 1
75
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
76
  - name: Install Libraries
 
77
  run: pip install -r requirements.txt
78
 
79
  - name: Download Artifact (datasets)
 
80
  uses: actions/download-artifact@v3
81
  with:
82
  name: datasets
 
91
  python setup.py build_ext --inplace && cd ..
92
 
93
  - name: Modeling and Training
 
94
  run: |
95
  unzip datasets.zip
96
 
 
99
  mkdir posttrained
100
 
101
  python trainingcli.py \
102
+ --epochs=200 \
103
+ --batchs=32 \
104
+ --sequences=60 \
105
  --algorithm=LSTM
 
 
 
 
106
 
107
  - name: Zip Posttrained, Models, and Pickles
 
108
  run: |
109
  zip -r models.zip models
110
  zip -r pickles.zip pickles
111
  zip -r posttrained.zip posttrained
112
 
113
+ - name: Store Models to Google Drive
 
114
  uses: adityak74/google-drive-upload-git-action@main
115
  with:
116
  credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
 
119
  name: models.zip
120
  overwrite: "true"
121
 
122
+ - name: Store Pickles to Google Drive
 
123
  uses: adityak74/google-drive-upload-git-action@main
124
  with:
125
  credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
 
128
  name: pickles.zip
129
  overwrite: "true"
130
 
131
+ - name: Store Posttrained to Google Drive
 
132
  uses: adityak74/google-drive-upload-git-action@main
133
  with:
134
  credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
 
138
  overwrite: "true"
139
 
140
  - name: Remove Temporarary Files and Directories
 
141
  run: |
142
  rm models.zip
143
  rm pickles.zip
 
149
  rm -rf datasets
150
  rm -rf posttrained
151
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
152
 
153
  tebakaja_crypto_space-0:
154
  name: crypto-forecast-svc-0