workflow

.github/workflows/checking_kaggle_status.sh

@@ -4,10 +4,10 @@
 
 check_status(){
     COMMAND="kaggle k status $1"
+    echo "EXECUTING: $COMMAND"
     while true
     do
-        echo "EXECUTING: $COMMAND"
-        sleep 2s
+        sleep 30s
         KERNEL_STATUS=$($COMMAND)
         echo "OUTPUT: $KERNEL_STATUS"
         # okay now try to exit or raise error
@@ -15,7 +15,7 @@ check_status(){
             # error
             echo "ERROR"
             exit 1
-            
+
         elif [[ $KERNEL_STATUS == *"has status \"complete\""* ]]; then
             # success
             echo "Notebook finished running !"

.github/workflows/kaggle.yaml

@@ -9,14 +9,26 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+
+      - name: Setup Kaggle
+        run: |
+          pip install kaggle
+          mkdir ~/.kaggle
+          echo "{\"username\":\"${{secrets.KAGGLE_USERNAME}}\",\"key\":\"${{secrets.KAGGLE_KEY}}\"}" > ~/.kaggle/kaggle.json
+          chmod 600 /home/runner/.kaggle/kaggle.json
+
       - name: Push Kaggle kernel
         run: kaggle kernels push -p kaggle/
   
       - name: Checking Kaggle Status
         run: |
-          bash .github/workflows/checking_kaggle_status.sh $KAGGLE_KERNEL_ID
+          bash .github/workflows/checking_kaggle_status.sh ${{ secrets.KAGGLE_KERNEL_ID }}
   
       - name: Download outputs
         run: |
-          kaggle k output $KAGGLE_KERNEL_ID -p outputs/
+          kaggle k output ${{ secrets.KAGGLE_KERNEL_ID }} -p outputs/ --quiet
           ls outputs/

README.md

@@ -79,7 +79,7 @@ Kaggle API docs:- https://github.com/Kaggle/kaggle-api/blob/main/docs/README.md
 ## Kaggle Commands:-
 - kaggle kernels pull anujpanthri/training-image-colorization-model -p kaggle/
 - kaggle kernels push -p kaggle/
-- echo "{\"username\":\"$(KAGGLE_USERNAME)\",\"key\":\"$KAGGLE_KEY\"}" > kaggle.json
+- echo "{\"username\":\"$KAGGLE_USERNAME\",\"key\":\"$KAGGLE_KEY\"}" > kaggle.json
 ### Version 1:
 
 - im gonna skip logging for now and rather use print statements

kaggle/train-image-colorization.ipynb

@@ -1 +1 @@
-{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:57:35.423893Z","iopub.status.busy":"2024-05-07T05:57:35.422571Z","iopub.status.idle":"2024-05-07T05:57:37.744423Z","shell.execute_reply":"2024-05-07T05:57:37.743082Z","shell.execute_reply.started":"2024-05-07T05:57:35.423838Z"},"trusted":true},"outputs":[],"source":["!git clone https://github.com/AnujPanthri/Image-Colorization.git\n","%cd Image-Colorization"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:57:38.983147Z","iopub.status.busy":"2024-05-07T05:57:38.982736Z","iopub.status.idle":"2024-05-07T05:57:38.989575Z","shell.execute_reply":"2024-05-07T05:57:38.987925Z","shell.execute_reply.started":"2024-05-07T05:57:38.983115Z"},"trusted":true},"outputs":[],"source":["# %cd ..\n","# !rm -r Image-Colorization"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:57:39.454828Z","iopub.status.busy":"2024-05-07T05:57:39.454444Z","iopub.status.idle":"2024-05-07T05:58:13.154377Z","shell.execute_reply":"2024-05-07T05:58:13.152964Z","shell.execute_reply.started":"2024-05-07T05:57:39.4548Z"},"trusted":true},"outputs":[],"source":["!pip install -r requirements.txt --quiet\n","!pip install -e ."]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:13.15798Z","iopub.status.busy":"2024-05-07T05:58:13.157513Z","iopub.status.idle":"2024-05-07T05:58:14.401676Z","shell.execute_reply":"2024-05-07T05:58:14.400055Z","shell.execute_reply.started":"2024-05-07T05:58:13.157935Z"},"trusted":true},"outputs":[],"source":["from kaggle_secrets import UserSecretsClient\n","user_secrets = UserSecretsClient()\n","COMET_API_KEY = user_secrets.get_secret(\"comet_api_key\")\n","\n","# !export COMET_API_KEY={COMET_API_KEY}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:14.412388Z","iopub.status.busy":"2024-05-07T05:58:14.412023Z","iopub.status.idle":"2024-05-07T05:58:14.422002Z","shell.execute_reply":"2024-05-07T05:58:14.420946Z","shell.execute_reply.started":"2024-05-07T05:58:14.41236Z"},"trusted":true},"outputs":[],"source":["import os\n","\n","os.environ[\"COMET_API_KEY\"]=COMET_API_KEY\n","config_file = \"configs/experiment1.yaml\""]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:14.423858Z","iopub.status.busy":"2024-05-07T05:58:14.423359Z","iopub.status.idle":"2024-05-07T05:58:20.695681Z","shell.execute_reply":"2024-05-07T05:58:20.694312Z","shell.execute_reply.started":"2024-05-07T05:58:14.423828Z"},"trusted":true},"outputs":[],"source":["!python3 src/scripts/prepare_dataset.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:20.697948Z","iopub.status.busy":"2024-05-07T05:58:20.697592Z","iopub.status.idle":"2024-05-07T05:58:20.703728Z","shell.execute_reply":"2024-05-07T05:58:20.702527Z","shell.execute_reply.started":"2024-05-07T05:58:20.697915Z"},"trusted":true},"outputs":[],"source":["# !cat src/utils/data_utils.py"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:24.072758Z","iopub.status.busy":"2024-05-07T05:58:24.072343Z","iopub.status.idle":"2024-05-07T05:58:27.518713Z","shell.execute_reply":"2024-05-07T05:58:27.517194Z","shell.execute_reply.started":"2024-05-07T05:58:24.072725Z"},"trusted":true},"outputs":[],"source":["# !python3 src/scripts/visualize_dataset.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:41.523718Z","iopub.status.busy":"2024-05-07T05:58:41.523307Z","iopub.status.idle":"2024-05-07T06:03:03.088039Z","shell.execute_reply":"2024-05-07T06:03:03.086519Z","shell.execute_reply.started":"2024-05-07T05:58:41.523683Z"},"trusted":true},"outputs":[],"source":["!python3 src/scripts/train.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T06:03:34.595185Z","iopub.status.busy":"2024-05-07T06:03:34.594725Z","iopub.status.idle":"2024-05-07T06:03:46.842693Z","shell.execute_reply":"2024-05-07T06:03:46.841521Z","shell.execute_reply.started":"2024-05-07T06:03:34.595147Z"},"trusted":true},"outputs":[],"source":["# !python3 src/scripts/visualize_results.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":30698,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.1"}},"nbformat":4,"nbformat_minor":4}
+{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:57:35.423893Z","iopub.status.busy":"2024-05-07T05:57:35.422571Z","iopub.status.idle":"2024-05-07T05:57:37.744423Z","shell.execute_reply":"2024-05-07T05:57:37.743082Z","shell.execute_reply.started":"2024-05-07T05:57:35.423838Z"},"trusted":true},"outputs":[],"source":["!git clone https://github.com/AnujPanthri/Image-Colorization.git\n","%cd Image-Colorization"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:57:38.983147Z","iopub.status.busy":"2024-05-07T05:57:38.982736Z","iopub.status.idle":"2024-05-07T05:57:38.989575Z","shell.execute_reply":"2024-05-07T05:57:38.987925Z","shell.execute_reply.started":"2024-05-07T05:57:38.983115Z"},"trusted":true},"outputs":[],"source":["# %cd ..\n","# !rm -r Image-Colorization"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:57:39.454828Z","iopub.status.busy":"2024-05-07T05:57:39.454444Z","iopub.status.idle":"2024-05-07T05:58:13.154377Z","shell.execute_reply":"2024-05-07T05:58:13.152964Z","shell.execute_reply.started":"2024-05-07T05:57:39.4548Z"},"trusted":true},"outputs":[],"source":["!pip install -r requirements.txt --quiet\n","!pip install -e ."]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:13.15798Z","iopub.status.busy":"2024-05-07T05:58:13.157513Z","iopub.status.idle":"2024-05-07T05:58:14.401676Z","shell.execute_reply":"2024-05-07T05:58:14.400055Z","shell.execute_reply.started":"2024-05-07T05:58:13.157935Z"},"trusted":true},"outputs":[],"source":["from kaggle_secrets import UserSecretsClient\n","user_secrets = UserSecretsClient()\n","COMET_API_KEY = user_secrets.get_secret(\"comet_api_key\")\n","\n","# !export COMET_API_KEY={COMET_API_KEY}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:14.412388Z","iopub.status.busy":"2024-05-07T05:58:14.412023Z","iopub.status.idle":"2024-05-07T05:58:14.422002Z","shell.execute_reply":"2024-05-07T05:58:14.420946Z","shell.execute_reply.started":"2024-05-07T05:58:14.41236Z"},"trusted":true},"outputs":[],"source":["import os\n","\n","os.environ[\"COMET_API_KEY\"]=COMET_API_KEY\n","config_file = \"configs/experiment1.yaml\""]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:14.423858Z","iopub.status.busy":"2024-05-07T05:58:14.423359Z","iopub.status.idle":"2024-05-07T05:58:20.695681Z","shell.execute_reply":"2024-05-07T05:58:20.694312Z","shell.execute_reply.started":"2024-05-07T05:58:14.423828Z"},"trusted":true},"outputs":[],"source":["!python3 src/scripts/prepare_dataset.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:20.697948Z","iopub.status.busy":"2024-05-07T05:58:20.697592Z","iopub.status.idle":"2024-05-07T05:58:20.703728Z","shell.execute_reply":"2024-05-07T05:58:20.702527Z","shell.execute_reply.started":"2024-05-07T05:58:20.697915Z"},"trusted":true},"outputs":[],"source":["# !cat src/utils/data_utils.py"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:24.072758Z","iopub.status.busy":"2024-05-07T05:58:24.072343Z","iopub.status.idle":"2024-05-07T05:58:27.518713Z","shell.execute_reply":"2024-05-07T05:58:27.517194Z","shell.execute_reply.started":"2024-05-07T05:58:24.072725Z"},"trusted":true},"outputs":[],"source":["# !python3 src/scripts/visualize_dataset.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:41.523718Z","iopub.status.busy":"2024-05-07T05:58:41.523307Z","iopub.status.idle":"2024-05-07T06:03:03.088039Z","shell.execute_reply":"2024-05-07T06:03:03.086519Z","shell.execute_reply.started":"2024-05-07T05:58:41.523683Z"},"trusted":true},"outputs":[],"source":["!python3 src/scripts/train.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T06:03:34.595185Z","iopub.status.busy":"2024-05-07T06:03:34.594725Z","iopub.status.idle":"2024-05-07T06:03:46.842693Z","shell.execute_reply":"2024-05-07T06:03:46.841521Z","shell.execute_reply.started":"2024-05-07T06:03:34.595147Z"},"trusted":true},"outputs":[],"source":["# !python3 src/scripts/visualize_results.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["%cd ..\n","\n","! cp -r Image-Colorization/artifacts .\n","! rm -r artifacts/model/\n","! rm -r Image-Colorization"]}],"metadata":{"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":30698,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.1"}},"nbformat":4,"nbformat_minor":4}