AnujPanthri commited on
Commit
0c2e200
·
2 Parent(s): df106d2 273ffa7

Merge branch 'train'

Browse files

get the latest pull from local train branch

download_model_comet.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from comet_ml.api import API
2
+ from src.utils.config_loader import constants
3
+
4
+ api = API()
5
+
6
+ # set env variable COMET_API_KEY
7
+
8
+ api.download_registry_model(
9
+ "anujpanthri",
10
+ "image-colorization-model",
11
+ version="1.0.0",
12
+ output_path=constants.ARTIFACT_MODEL_DIR,
13
+ expand=True,
14
+ stage=None,
15
+ )
kaggle/train-image-colorization.ipynb CHANGED
@@ -1 +1 @@
1
- {"cells":[{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:57:35.423893Z","iopub.status.busy":"2024-05-07T05:57:35.422571Z","iopub.status.idle":"2024-05-07T05:57:37.744423Z","shell.execute_reply":"2024-05-07T05:57:37.743082Z","shell.execute_reply.started":"2024-05-07T05:57:35.423838Z"},"trusted":true},"outputs":[],"source":["!git clone https://github.com/AnujPanthri/Image-Colorization.git\n","%cd Image-Colorization\n","!git checkout train"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:57:38.983147Z","iopub.status.busy":"2024-05-07T05:57:38.982736Z","iopub.status.idle":"2024-05-07T05:57:38.989575Z","shell.execute_reply":"2024-05-07T05:57:38.987925Z","shell.execute_reply.started":"2024-05-07T05:57:38.983115Z"},"trusted":true},"outputs":[],"source":["# %cd ..\n","# !rm -r Image-Colorization"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:57:39.454828Z","iopub.status.busy":"2024-05-07T05:57:39.454444Z","iopub.status.idle":"2024-05-07T05:58:13.154377Z","shell.execute_reply":"2024-05-07T05:58:13.152964Z","shell.execute_reply.started":"2024-05-07T05:57:39.4548Z"},"trusted":true},"outputs":[],"source":["!pip install -r requirements.txt --quiet\n","!pip install -e ."]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:13.15798Z","iopub.status.busy":"2024-05-07T05:58:13.157513Z","iopub.status.idle":"2024-05-07T05:58:14.401676Z","shell.execute_reply":"2024-05-07T05:58:14.400055Z","shell.execute_reply.started":"2024-05-07T05:58:13.157935Z"},"trusted":true},"outputs":[],"source":["from kaggle_secrets import UserSecretsClient\n","user_secrets = UserSecretsClient()\n","COMET_API_KEY = user_secrets.get_secret(\"comet_api_key\")\n","\n","# !export COMET_API_KEY={COMET_API_KEY}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:14.412388Z","iopub.status.busy":"2024-05-07T05:58:14.412023Z","iopub.status.idle":"2024-05-07T05:58:14.422002Z","shell.execute_reply":"2024-05-07T05:58:14.420946Z","shell.execute_reply.started":"2024-05-07T05:58:14.41236Z"},"trusted":true},"outputs":[],"source":["import os\n","\n","os.environ[\"COMET_API_KEY\"]=COMET_API_KEY\n","config_file = \"configs/experiment1.yaml\""]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:14.423858Z","iopub.status.busy":"2024-05-07T05:58:14.423359Z","iopub.status.idle":"2024-05-07T05:58:20.695681Z","shell.execute_reply":"2024-05-07T05:58:20.694312Z","shell.execute_reply.started":"2024-05-07T05:58:14.423828Z"},"trusted":true},"outputs":[],"source":["!python3 src/scripts/prepare_dataset.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:20.697948Z","iopub.status.busy":"2024-05-07T05:58:20.697592Z","iopub.status.idle":"2024-05-07T05:58:20.703728Z","shell.execute_reply":"2024-05-07T05:58:20.702527Z","shell.execute_reply.started":"2024-05-07T05:58:20.697915Z"},"trusted":true},"outputs":[],"source":["# !cat src/utils/data_utils.py"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:24.072758Z","iopub.status.busy":"2024-05-07T05:58:24.072343Z","iopub.status.idle":"2024-05-07T05:58:27.518713Z","shell.execute_reply":"2024-05-07T05:58:27.517194Z","shell.execute_reply.started":"2024-05-07T05:58:24.072725Z"},"trusted":true},"outputs":[],"source":["!python3 src/scripts/visualize_dataset.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:41.523718Z","iopub.status.busy":"2024-05-07T05:58:41.523307Z","iopub.status.idle":"2024-05-07T06:03:03.088039Z","shell.execute_reply":"2024-05-07T06:03:03.086519Z","shell.execute_reply.started":"2024-05-07T05:58:41.523683Z"},"trusted":true},"outputs":[],"source":["!python3 src/scripts/train.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T06:03:34.595185Z","iopub.status.busy":"2024-05-07T06:03:34.594725Z","iopub.status.idle":"2024-05-07T06:03:46.842693Z","shell.execute_reply":"2024-05-07T06:03:46.841521Z","shell.execute_reply.started":"2024-05-07T06:03:34.595147Z"},"trusted":true},"outputs":[],"source":["!python3 src/scripts/visualize_results.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["%cd ..\n","\n","! cp -r Image-Colorization/artifacts .\n","! rm -r artifacts/model/\n","! rm -r Image-Colorization\n","! ls"]}],"metadata":{"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":30698,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.1"}},"nbformat":4,"nbformat_minor":4}
 
1
+ {"cells":[{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:57:35.423893Z","iopub.status.busy":"2024-05-07T05:57:35.422571Z","iopub.status.idle":"2024-05-07T05:57:37.744423Z","shell.execute_reply":"2024-05-07T05:57:37.743082Z","shell.execute_reply.started":"2024-05-07T05:57:35.423838Z"},"trusted":true},"outputs":[],"source":["!git clone https://github.com/AnujPanthri/Image-Colorization.git\n","%cd Image-Colorization\n","!git checkout train"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:57:38.983147Z","iopub.status.busy":"2024-05-07T05:57:38.982736Z","iopub.status.idle":"2024-05-07T05:57:38.989575Z","shell.execute_reply":"2024-05-07T05:57:38.987925Z","shell.execute_reply.started":"2024-05-07T05:57:38.983115Z"},"trusted":true},"outputs":[],"source":["# %cd ..\n","# !rm -r Image-Colorization"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:57:39.454828Z","iopub.status.busy":"2024-05-07T05:57:39.454444Z","iopub.status.idle":"2024-05-07T05:58:13.154377Z","shell.execute_reply":"2024-05-07T05:58:13.152964Z","shell.execute_reply.started":"2024-05-07T05:57:39.4548Z"},"trusted":true},"outputs":[],"source":["!pip install -r requirements.txt --quiet\n","!pip install -e ."]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:13.15798Z","iopub.status.busy":"2024-05-07T05:58:13.157513Z","iopub.status.idle":"2024-05-07T05:58:14.401676Z","shell.execute_reply":"2024-05-07T05:58:14.400055Z","shell.execute_reply.started":"2024-05-07T05:58:13.157935Z"},"trusted":true},"outputs":[],"source":["from kaggle_secrets import UserSecretsClient\n","user_secrets = UserSecretsClient()\n","COMET_API_KEY = user_secrets.get_secret(\"comet_api_key\")\n","\n","# !export COMET_API_KEY={COMET_API_KEY}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:14.412388Z","iopub.status.busy":"2024-05-07T05:58:14.412023Z","iopub.status.idle":"2024-05-07T05:58:14.422002Z","shell.execute_reply":"2024-05-07T05:58:14.420946Z","shell.execute_reply.started":"2024-05-07T05:58:14.41236Z"},"trusted":true},"outputs":[],"source":["import os\n","\n","os.environ[\"COMET_API_KEY\"]=COMET_API_KEY\n","config_file = \"configs/experiment1.yaml\""]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:14.423858Z","iopub.status.busy":"2024-05-07T05:58:14.423359Z","iopub.status.idle":"2024-05-07T05:58:20.695681Z","shell.execute_reply":"2024-05-07T05:58:20.694312Z","shell.execute_reply.started":"2024-05-07T05:58:14.423828Z"},"trusted":true},"outputs":[],"source":["!python3 src/scripts/prepare_dataset.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:20.697948Z","iopub.status.busy":"2024-05-07T05:58:20.697592Z","iopub.status.idle":"2024-05-07T05:58:20.703728Z","shell.execute_reply":"2024-05-07T05:58:20.702527Z","shell.execute_reply.started":"2024-05-07T05:58:20.697915Z"},"trusted":true},"outputs":[],"source":["# !cat src/utils/data_utils.py"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:24.072758Z","iopub.status.busy":"2024-05-07T05:58:24.072343Z","iopub.status.idle":"2024-05-07T05:58:27.518713Z","shell.execute_reply":"2024-05-07T05:58:27.517194Z","shell.execute_reply.started":"2024-05-07T05:58:24.072725Z"},"trusted":true},"outputs":[],"source":["!python3 src/scripts/visualize_dataset.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T05:58:41.523718Z","iopub.status.busy":"2024-05-07T05:58:41.523307Z","iopub.status.idle":"2024-05-07T06:03:03.088039Z","shell.execute_reply":"2024-05-07T06:03:03.086519Z","shell.execute_reply.started":"2024-05-07T05:58:41.523683Z"},"trusted":true},"outputs":[],"source":["!python3 src/scripts/train.py {config_file} --log"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-05-07T06:03:34.595185Z","iopub.status.busy":"2024-05-07T06:03:34.594725Z","iopub.status.idle":"2024-05-07T06:03:46.842693Z","shell.execute_reply":"2024-05-07T06:03:46.841521Z","shell.execute_reply.started":"2024-05-07T06:03:34.595147Z"},"trusted":true},"outputs":[],"source":["!python3 src/scripts/visualize_results.py {config_file}"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["%cd ..\n","\n","! cp -r Image-Colorization/artifacts .\n","! rm -r artifacts/model/\n","! rm -r Image-Colorization\n","! ls"]}],"metadata":{"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":30698,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.1"}},"nbformat":4,"nbformat_minor":4}
requirements.txt CHANGED
@@ -1,4 +1,77 @@
1
- huggingface_hub
2
- comet_ml
3
- scikit-image
4
- cerberus
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==2.1.0
2
+ astunparse==1.6.3
3
+ attrs==23.2.0
4
+ Cerberus==1.3.5
5
+ certifi==2024.2.2
6
+ charset-normalizer==3.3.2
7
+ comet-ml==3.42.1
8
+ configobj==5.0.8
9
+ contourpy==1.2.1
10
+ cycler==0.12.1
11
+ dulwich==0.22.1
12
+ everett==3.1.0
13
+ filelock==3.14.0
14
+ flatbuffers==24.3.25
15
+ fonttools==4.51.0
16
+ fsspec==2024.5.0
17
+ gast==0.5.4
18
+ google-pasta==0.2.0
19
+ grpcio==1.64.0
20
+ h5py==3.11.0
21
+ huggingface-hub==0.23.1
22
+ idna==3.7
23
+ imageio==2.34.1
24
+ joblib==1.4.2
25
+ jsonschema==4.22.0
26
+ jsonschema-specifications==2023.12.1
27
+ keras==3.3.3
28
+ kiwisolver==1.4.5
29
+ lazy_loader==0.4
30
+ libclang==18.1.1
31
+ Markdown==3.6
32
+ markdown-it-py==3.0.0
33
+ MarkupSafe==2.1.5
34
+ matplotlib==3.9.0
35
+ mdurl==0.1.2
36
+ ml-dtypes==0.3.2
37
+ namex==0.0.8
38
+ networkx==3.3
39
+ numpy==1.26.4
40
+ opencv-python==4.9.0.80
41
+ opt-einsum==3.3.0
42
+ optree==0.11.0
43
+ packaging==24.0
44
+ pillow==10.3.0
45
+ protobuf==4.25.3
46
+ psutil==5.9.8
47
+ Pygments==2.18.0
48
+ pyparsing==3.1.2
49
+ python-box==6.1.0
50
+ python-dateutil==2.9.0.post0
51
+ PyYAML==6.0.1
52
+ referencing==0.35.1
53
+ requests==2.32.2
54
+ requests-toolbelt==1.0.0
55
+ rich==13.7.1
56
+ rpds-py==0.18.1
57
+ scikit-image==0.23.2
58
+ scikit-learn==1.5.0
59
+ scipy==1.13.1
60
+ semantic-version==2.10.0
61
+ sentry-sdk==2.2.1
62
+ setuptools==69.2.0
63
+ simplejson==3.19.2
64
+ six==1.16.0
65
+ tensorboard==2.16.2
66
+ tensorboard-data-server==0.7.2
67
+ tensorflow==2.16.1
68
+ termcolor==2.4.0
69
+ threadpoolctl==3.5.0
70
+ tifffile==2024.5.10
71
+ tqdm==4.66.4
72
+ typing_extensions==4.11.0
73
+ urllib3==2.2.1
74
+ Werkzeug==3.0.3
75
+ wheel==0.43.0
76
+ wrapt==1.16.0
77
+ wurlitzer==3.1.0
src/scripts/train.py CHANGED
@@ -1,7 +1,7 @@
1
  import os,shutil
2
  import argparse
3
  from comet_ml import Experiment
4
- from src.utils.config_loader import Config,constants
5
  from src.utils import config_loader
6
  from src.utils.data_utils import print_title
7
  from src.utils.script_utils import validate_config
@@ -17,8 +17,10 @@ def train(args):
17
  # validate config
18
  validate_config(config)
19
 
20
- # set config globally
21
  config_loader.config = config
 
 
22
 
23
  # now load the model
24
  Model = importlib.import_module(f"src.{config.task}.model.models.{config.model}").Model
@@ -55,7 +57,7 @@ def train(args):
55
  # log model to comet
56
  if "LOCAL_SYSTEM" not in os.environ:
57
  if experiment:
58
- experiment.log_model(f"{config.task}_{config.dataset}_{config.model}",model_dir)
59
 
60
  # evaluate model
61
  print_title("\nEvaluating Model")
 
1
  import os,shutil
2
  import argparse
3
  from comet_ml import Experiment
4
+ from src.utils.config_loader import Config,constants,set_seed
5
  from src.utils import config_loader
6
  from src.utils.data_utils import print_title
7
  from src.utils.script_utils import validate_config
 
17
  # validate config
18
  validate_config(config)
19
 
20
+ # set config globally & set seed
21
  config_loader.config = config
22
+ set_seed(config.seed)
23
+
24
 
25
  # now load the model
26
  Model = importlib.import_module(f"src.{config.task}.model.models.{config.model}").Model
 
57
  # log model to comet
58
  if "LOCAL_SYSTEM" not in os.environ:
59
  if experiment:
60
+ experiment.log_model(f"model",model_dir)
61
 
62
  # evaluate model
63
  print_title("\nEvaluating Model")
src/scripts/visualize_dataset.py CHANGED
@@ -1,8 +1,9 @@
1
  import argparse
2
- from src.utils.config_loader import Config
3
  from src.utils import config_loader
4
  from src.utils.script_utils import validate_config
5
  import importlib
 
6
 
7
 
8
  def visualize_dataset(args):
@@ -12,8 +13,9 @@ def visualize_dataset(args):
12
  # validate config
13
  validate_config(config)
14
 
15
- # set config globally
16
  config_loader.config = config
 
17
 
18
  # now visualize the dataset
19
  visualize_fn = importlib.import_module(f"src.{config.task}.data.visualize_dataset").visualize
 
1
  import argparse
2
+ from src.utils.config_loader import Config,set_seed
3
  from src.utils import config_loader
4
  from src.utils.script_utils import validate_config
5
  import importlib
6
+ import random
7
 
8
 
9
  def visualize_dataset(args):
 
13
  # validate config
14
  validate_config(config)
15
 
16
+ # set config globally and set seed
17
  config_loader.config = config
18
+ set_seed(config.seed)
19
 
20
  # now visualize the dataset
21
  visualize_fn = importlib.import_module(f"src.{config.task}.data.visualize_dataset").visualize
src/scripts/visualize_results.py CHANGED
@@ -1,6 +1,6 @@
1
  import os
2
  import argparse
3
- from src.utils.config_loader import Config,constants
4
  from src.utils import config_loader
5
  from src.utils.script_utils import validate_config
6
  import importlib
@@ -13,8 +13,9 @@ def visualize_results(args):
13
  # validate config
14
  validate_config(config)
15
 
16
- # set config globally
17
  config_loader.config = config
 
18
 
19
  # now load model and visualize the results
20
  model_dir = constants.ARTIFACT_MODEL_DIR
 
1
  import os
2
  import argparse
3
+ from src.utils.config_loader import Config,constants,set_seed
4
  from src.utils import config_loader
5
  from src.utils.script_utils import validate_config
6
  import importlib
 
13
  # validate config
14
  validate_config(config)
15
 
16
+ # set config globally & set seed
17
  config_loader.config = config
18
+ set_seed(config.seed)
19
 
20
  # now load model and visualize the results
21
  model_dir = constants.ARTIFACT_MODEL_DIR
src/utils/config_loader.py CHANGED
@@ -1,5 +1,9 @@
1
  import yaml
2
  from pathlib import Path
 
 
 
 
3
 
4
  class Config:
5
  def __init__(self,config_file_path:str):
@@ -20,4 +24,18 @@ constants.config_dict['RAW_DATASET_DIR'] = Path(constants.config_dict['RAW_DATAS
20
  constants.config_dict['INTERIM_DATASET_DIR'] = Path(constants.config_dict['INTERIM_DATASET_DIR'])
21
  constants.config_dict['PROCESSED_DATASET_DIR'] = Path(constants.config_dict['PROCESSED_DATASET_DIR'])
22
 
23
- config = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import yaml
2
  from pathlib import Path
3
+ import random
4
+ import tensorflow as tf
5
+ import numpy as np
6
+ import os
7
 
8
  class Config:
9
  def __init__(self,config_file_path:str):
 
24
  constants.config_dict['INTERIM_DATASET_DIR'] = Path(constants.config_dict['INTERIM_DATASET_DIR'])
25
  constants.config_dict['PROCESSED_DATASET_DIR'] = Path(constants.config_dict['PROCESSED_DATASET_DIR'])
26
 
27
+ config = None
28
+
29
+
30
+ def set_seed(seed: int = 42) -> None:
31
+ random.seed(seed)
32
+ np.random.seed(seed)
33
+ tf.random.set_seed(seed)
34
+ tf.experimental.numpy.random.seed(seed)
35
+
36
+ # When running on the CuDNN backend, two further options must be set
37
+ os.environ['TF_CUDNN_DETERMINISTIC'] = '1'
38
+ os.environ['TF_DETERMINISTIC_OPS'] = '1'
39
+ # Set a fixed value for the hash seed
40
+ os.environ["PYTHONHASHSEED"] = str(seed)
41
+ print(f"Random seed set as {seed}")