|
name: GRU Pipeline |
|
|
|
on: |
|
push: |
|
branches: |
|
- main |
|
tags: |
|
- '*' |
|
schedule: |
|
- cron: "0 7 * * *" |
|
|
|
|
|
jobs: |
|
scraping_extracting: |
|
name: Scraping and Extracting |
|
runs-on: ubuntu-latest |
|
environment: Scraping and Extraction |
|
|
|
steps: |
|
- name: Set global directory |
|
run: git config --global --add safe.directory /github/workspace |
|
|
|
- uses: actions/checkout@v3 |
|
with: |
|
lfs: true |
|
persist-credentials: false |
|
fetch-depth: 1 |
|
|
|
- name: Read pipeline schedule date |
|
id: read_schedule |
|
run: | |
|
SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl) |
|
echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV |
|
|
|
- name: Get current date |
|
id: get_date |
|
run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV |
|
|
|
- name: Check if dates match |
|
id: date_check |
|
run: | |
|
if [ "$schedule_date" = "$current_date" ]; then |
|
echo "match=true" >> $GITHUB_ENV |
|
else |
|
echo "match=false" >> $GITHUB_ENV |
|
fi |
|
|
|
- name: Scraping Yahoo Finance |
|
if: env.match != 'true' |
|
run: | |
|
mkdir datasets |
|
go run scraper.go \ |
|
--symbols-file=./postman/symbols_test.json |
|
|
|
- name: Zip Datasets |
|
if: env.match != 'true' |
|
run: zip -r datasets.zip datasets |
|
|
|
- name: Store Datasets to Google Drive |
|
if: env.match != 'true' |
|
uses: adityak74/google-drive-upload-git-action@main |
|
with: |
|
credentials: ${{ secrets.GDRIVE_GRU_CRED }} |
|
filename: datasets.zip |
|
folderId: ${{ secrets.GDRIVE_GRU_ID }} |
|
name: datasets.zip |
|
overwrite: "true" |
|
|
|
- name: Upload Artifact (datasets) |
|
if: env.match != 'true' |
|
uses: actions/upload-artifact@v3 |
|
with: |
|
name: datasets |
|
path: datasets.zip |
|
|
|
- name: Remove Temporarary Files and Directories |
|
if: env.match != 'true' |
|
run: | |
|
rm datasets.zip |
|
rm -rf datasets |
|
|
|
|
|
preprocessing_training: |
|
name: Preprocessing and Training |
|
runs-on: ubuntu-latest |
|
needs: scraping_extracting |
|
environment: Preprocessing and Training |
|
|
|
steps: |
|
- name: Set global directory |
|
run: git config --global --add safe.directory /github/workspace |
|
|
|
- uses: actions/checkout@v3 |
|
with: |
|
lfs: true |
|
persist-credentials: false |
|
fetch-depth: 1 |
|
|
|
- name: Read pipeline schedule date |
|
id: read_schedule |
|
run: | |
|
SCHEDULE_DATE=$(cat schedulers/gru_schedule.ctl) |
|
echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV |
|
|
|
- name: Get current date |
|
id: get_date |
|
run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV |
|
|
|
- name: Check if dates match |
|
id: date_check |
|
run: | |
|
if [ "$schedule_date" = "$current_date" ]; then |
|
echo "match=true" >> $GITHUB_ENV |
|
else |
|
echo "match=false" >> $GITHUB_ENV |
|
fi |
|
|
|
- name: Install Libraries |
|
if: env.match != 'true' |
|
run: pip install -r requirements.txt |
|
|
|
- name: Download Artifact (datasets) |
|
if: env.match != 'true' |
|
uses: actions/download-artifact@v3 |
|
with: |
|
name: datasets |
|
|
|
- name: Modeling and Training |
|
if: env.match != 'true' |
|
run: | |
|
unzip datasets.zip |
|
|
|
mkdir models |
|
mkdir pickles |
|
mkdir posttrained |
|
|
|
python training.py \ |
|
--epochs=200 \ |
|
--batchs=32 \ |
|
--sequences=5 \ |
|
--algorithm=GRU |
|
|
|
- name: Set Pipeline Schedule |
|
if: env.match != 'true' |
|
run: echo "$(date +'%Y-%m-%d')" > schedulers/gru_schedule.ctl |
|
|
|
- name: Zip Posttrained, Models, and Pickles |
|
if: env.match != 'true' |
|
run: | |
|
zip -r models.zip models |
|
zip -r pickles.zip pickles |
|
zip -r posttrained.zip posttrained |
|
|
|
- name: Store Models to Google Drive |
|
if: env.match != 'true' |
|
uses: adityak74/google-drive-upload-git-action@main |
|
with: |
|
credentials: ${{ secrets.GDRIVE_GRU_CRED }} |
|
filename: models.zip |
|
folderId: ${{ secrets.GDRIVE_GRU_ID }} |
|
name: models.zip |
|
overwrite: "true" |
|
|
|
- name: Store Pickles to Google Drive |
|
if: env.match != 'true' |
|
uses: adityak74/google-drive-upload-git-action@main |
|
with: |
|
credentials: ${{ secrets.GDRIVE_GRU_CRED }} |
|
filename: pickles.zip |
|
folderId: ${{ secrets.GDRIVE_GRU_ID }} |
|
name: pickles.zip |
|
overwrite: "true" |
|
|
|
- name: Store Posttrained to Google Drive |
|
if: env.match != 'true' |
|
uses: adityak74/google-drive-upload-git-action@main |
|
with: |
|
credentials: ${{ secrets.GDRIVE_GRU_CRED }} |
|
filename: posttrained.zip |
|
folderId: ${{ secrets.GDRIVE_GRU_ID }} |
|
name: posttrained.zip |
|
overwrite: "true" |
|
|
|
- name: Remove Temporarary Files and Directories |
|
if: env.match != 'true' |
|
run: | |
|
rm models.zip |
|
rm pickles.zip |
|
rm posttrained.zip |
|
|
|
rm -rf models |
|
rm -rf pickles |
|
rm -rf posttrained |
|
|
|
- name: Commit changes |
|
if: env.match != 'true' |
|
run: | |
|
git config --local user.email "[email protected]" |
|
git config --local user.name "belajarqywok" |
|
git add -A |
|
git commit -m "Data Extraction, Training, and Modeling" |
|
|
|
- name: Push changes |
|
if: env.match != 'true' |
|
uses: ad-m/github-push-action@master |
|
with: |
|
github_token: ${{ secrets.GH_TOKEN }} |
|
branch: main |
|
|
|
|
|
tebakaja_crypto_space-0: |
|
name: crypto-forecast-svc-0 |
|
runs-on: ubuntu-latest |
|
needs: preprocessing_training |
|
environment: Deployment |
|
|
|
env: |
|
HF_TOKEN: ${{ secrets.HF_TOKEN }} |
|
SPACE_NAME: tebakaja_cryptocurrency_space-0 |
|
HF_USERNAME: tebakaja |
|
|
|
steps: |
|
- name: Set global directory |
|
run: git config --global --add safe.directory /github/workspace |
|
|
|
- uses: actions/checkout@v3 |
|
with: |
|
persist-credentials: false |
|
fetch-depth: 1000 |
|
|
|
- name: Check git status |
|
run: git status |
|
|
|
- name: Configure git |
|
run: | |
|
git config --local user.email "[email protected]" |
|
git config --local user.name "qywok" |
|
|
|
- name: Pull changes from remote |
|
run: | |
|
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \ |
|
(git merge --strategy-option theirs) |
|
|
|
- name: Add and commit changes |
|
run: | |
|
git add -A |
|
git diff-index --quiet HEAD || git commit -m "Model Deployment" |
|
|
|
- name: Push to Hugging Face |
|
run: | |
|
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force |
|
|
|
|
|
tebakaja_crypto_space-1: |
|
name: crypto-forecast-svc-1 |
|
runs-on: ubuntu-latest |
|
needs: preprocessing_training |
|
environment: Deployment |
|
|
|
env: |
|
HF_TOKEN: ${{ secrets.HF_TOKEN }} |
|
SPACE_NAME: tebakaja_cryptocurrency_space-1 |
|
HF_USERNAME: tebakaja |
|
|
|
steps: |
|
- name: Set global directory |
|
run: git config --global --add safe.directory /github/workspace |
|
|
|
- uses: actions/checkout@v3 |
|
with: |
|
persist-credentials: false |
|
fetch-depth: 1000 |
|
|
|
- name: Check git status |
|
run: git status |
|
|
|
- name: Configure git |
|
run: | |
|
git config --local user.email "[email protected]" |
|
git config --local user.name "qywok" |
|
|
|
- name: Pull changes from remote |
|
run: | |
|
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \ |
|
(git merge --strategy-option theirs) |
|
|
|
- name: Add and commit changes |
|
run: | |
|
git add -A |
|
git diff-index --quiet HEAD || git commit -m "Model Deployment" |
|
|
|
- name: Push to Hugging Face |
|
run: | |
|
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force |
|
|
|
|
|
tebakaja_crypto_space-2: |
|
name: crypto-forecast-svc-2 |
|
runs-on: ubuntu-latest |
|
needs: preprocessing_training |
|
environment: Deployment |
|
|
|
env: |
|
HF_TOKEN: ${{ secrets.HF_TOKEN }} |
|
SPACE_NAME: tebakaja_cryptocurrency_space-2 |
|
HF_USERNAME: tebakaja |
|
|
|
steps: |
|
- name: Set global directory |
|
run: git config --global --add safe.directory /github/workspace |
|
|
|
- uses: actions/checkout@v3 |
|
with: |
|
persist-credentials: false |
|
fetch-depth: 1000 |
|
|
|
- name: Check git status |
|
run: git status |
|
|
|
- name: Configure git |
|
run: | |
|
git config --local user.email "[email protected]" |
|
git config --local user.name "qywok" |
|
|
|
- name: Pull changes from remote |
|
run: | |
|
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \ |
|
(git merge --strategy-option theirs) |
|
|
|
- name: Add and commit changes |
|
run: | |
|
git add -A |
|
git diff-index --quiet HEAD || git commit -m "Model Deployment" |
|
|
|
- name: Push to Hugging Face |
|
run: | |
|
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force |
|
|
|
|
|
tebakaja_crypto_space-3: |
|
name: crypto-forecast-svc-3 |
|
runs-on: ubuntu-latest |
|
needs: preprocessing_training |
|
environment: Deployment |
|
|
|
env: |
|
HF_TOKEN: ${{ secrets.HF_TOKEN }} |
|
SPACE_NAME: tebakaja_cryptocurrency_space-3 |
|
HF_USERNAME: tebakaja |
|
|
|
steps: |
|
- name: Set global directory |
|
run: git config --global --add safe.directory /github/workspace |
|
|
|
- uses: actions/checkout@v3 |
|
with: |
|
persist-credentials: false |
|
fetch-depth: 1000 |
|
|
|
- name: Check git status |
|
run: git status |
|
|
|
- name: Configure git |
|
run: | |
|
git config --local user.email "[email protected]" |
|
git config --local user.name "qywok" |
|
|
|
- name: Pull changes from remote |
|
run: | |
|
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \ |
|
(git merge --strategy-option theirs) |
|
|
|
- name: Add and commit changes |
|
run: | |
|
git add -A |
|
git diff-index --quiet HEAD || git commit -m "Model Deployment" |
|
|
|
- name: Push to Hugging Face |
|
run: | |
|
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force |
|
|
|
|
|
tebakaja_crypto_space-4: |
|
name: crypto-forecast-svc-4 |
|
runs-on: ubuntu-latest |
|
needs: preprocessing_training |
|
environment: Deployment |
|
|
|
env: |
|
HF_TOKEN: ${{ secrets.HF_TOKEN }} |
|
SPACE_NAME: tebakaja_cryptocurrency_space-4 |
|
HF_USERNAME: tebakaja |
|
|
|
steps: |
|
- name: Set global directory |
|
run: git config --global --add safe.directory /github/workspace |
|
|
|
- uses: actions/checkout@v3 |
|
with: |
|
persist-credentials: false |
|
fetch-depth: 1000 |
|
|
|
- name: Check git status |
|
run: git status |
|
|
|
- name: Configure git |
|
run: | |
|
git config --local user.email "[email protected]" |
|
git config --local user.name "qywok" |
|
|
|
- name: Pull changes from remote |
|
run: | |
|
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \ |
|
(git merge --strategy-option theirs) |
|
|
|
- name: Add and commit changes |
|
run: | |
|
git add -A |
|
git diff-index --quiet HEAD || git commit -m "Model Deployment" |
|
|
|
- name: Push to Hugging Face |
|
run: | |
|
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force |
|
|
|
|
|
functional_testing: |
|
name: Functional Testing |
|
runs-on: ubuntu-latest |
|
|
|
needs: |
|
- tebakaja_crypto_space-0 |
|
- tebakaja_crypto_space-1 |
|
- tebakaja_crypto_space-2 |
|
- tebakaja_crypto_space-3 |
|
- tebakaja_crypto_space-4 |
|
|
|
steps: |
|
- name: Set global directory |
|
run: git config --global --add safe.directory /github/workspace |
|
|
|
- uses: actions/checkout@v3 |
|
with: |
|
persist-credentials: false |
|
fetch-depth: 1 |
|
|
|
- name: Testing Proxy Endpoints |
|
run: | |
|
sleep 60 |
|
chmod +x endpoints_test.sh && ./endpoints_test.sh |
|
|
|
|