utils/wandb_logging PEP8 reformat (#2755)
Browse files* wandb_logging PEP8 reformat
* Update wandb_utils.py
utils/wandb_logging/log_dataset.py
CHANGED
@@ -1,10 +1,8 @@
|
|
1 |
import argparse
|
2 |
-
from pathlib import Path
|
3 |
|
4 |
import yaml
|
5 |
|
6 |
from wandb_utils import WandbLogger
|
7 |
-
from utils.datasets import LoadImagesAndLabels
|
8 |
|
9 |
WANDB_ARTIFACT_PREFIX = 'wandb-artifact://'
|
10 |
|
@@ -21,6 +19,6 @@ if __name__ == '__main__':
|
|
21 |
parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset')
|
22 |
parser.add_argument('--project', type=str, default='YOLOv5', help='name of W&B Project')
|
23 |
opt = parser.parse_args()
|
24 |
-
opt.resume = False
|
25 |
-
|
26 |
create_dataset_artifact(opt)
|
|
|
1 |
import argparse
|
|
|
2 |
|
3 |
import yaml
|
4 |
|
5 |
from wandb_utils import WandbLogger
|
|
|
6 |
|
7 |
WANDB_ARTIFACT_PREFIX = 'wandb-artifact://'
|
8 |
|
|
|
19 |
parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset')
|
20 |
parser.add_argument('--project', type=str, default='YOLOv5', help='name of W&B Project')
|
21 |
opt = parser.parse_args()
|
22 |
+
opt.resume = False # Explicitly disallow resume check for dataset upload job
|
23 |
+
|
24 |
create_dataset_artifact(opt)
|
utils/wandb_logging/wandb_utils.py
CHANGED
@@ -1,12 +1,9 @@
|
|
1 |
-
import argparse
|
2 |
import json
|
3 |
-
import os
|
4 |
-
import shutil
|
5 |
import sys
|
|
|
|
|
6 |
import torch
|
7 |
import yaml
|
8 |
-
from datetime import datetime
|
9 |
-
from pathlib import Path
|
10 |
from tqdm import tqdm
|
11 |
|
12 |
sys.path.append(str(Path(__file__).parent.parent.parent)) # add utils/ to path
|
@@ -33,6 +30,7 @@ def check_wandb_config_file(data_config_file):
|
|
33 |
return wandb_config
|
34 |
return data_config_file
|
35 |
|
|
|
36 |
def get_run_info(run_path):
|
37 |
run_path = Path(remove_prefix(run_path, WANDB_ARTIFACT_PREFIX))
|
38 |
run_id = run_path.stem
|
@@ -40,11 +38,12 @@ def get_run_info(run_path):
|
|
40 |
model_artifact_name = 'run_' + run_id + '_model'
|
41 |
return run_id, project, model_artifact_name
|
42 |
|
|
|
43 |
def check_wandb_resume(opt):
|
44 |
process_wandb_config_ddp_mode(opt) if opt.global_rank not in [-1, 0] else None
|
45 |
if isinstance(opt.resume, str):
|
46 |
if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):
|
47 |
-
if opt.global_rank not in [-1, 0]:
|
48 |
run_id, project, model_artifact_name = get_run_info(opt.resume)
|
49 |
api = wandb.Api()
|
50 |
artifact = api.artifact(project + '/' + model_artifact_name + ':latest')
|
@@ -53,6 +52,7 @@ def check_wandb_resume(opt):
|
|
53 |
return True
|
54 |
return None
|
55 |
|
|
|
56 |
def process_wandb_config_ddp_mode(opt):
|
57 |
with open(opt.data) as f:
|
58 |
data_dict = yaml.load(f, Loader=yaml.SafeLoader) # data dict
|
@@ -63,7 +63,7 @@ def process_wandb_config_ddp_mode(opt):
|
|
63 |
train_dir = train_artifact.download()
|
64 |
train_path = Path(train_dir) / 'data/images/'
|
65 |
data_dict['train'] = str(train_path)
|
66 |
-
|
67 |
if isinstance(data_dict['val'], str) and data_dict['val'].startswith(WANDB_ARTIFACT_PREFIX):
|
68 |
api = wandb.Api()
|
69 |
val_artifact = api.artifact(remove_prefix(data_dict['val']) + ':' + opt.artifact_alias)
|
@@ -71,12 +71,11 @@ def process_wandb_config_ddp_mode(opt):
|
|
71 |
val_path = Path(val_dir) / 'data/images/'
|
72 |
data_dict['val'] = str(val_path)
|
73 |
if train_dir or val_dir:
|
74 |
-
ddp_data_path = str(Path(val_dir) / 'wandb_local_data.yaml')
|
75 |
with open(ddp_data_path, 'w') as f:
|
76 |
yaml.dump(data_dict, f)
|
77 |
opt.data = ddp_data_path
|
78 |
-
|
79 |
-
|
80 |
|
81 |
class WandbLogger():
|
82 |
def __init__(self, opt, name, run_id, data_dict, job_type='Training'):
|
@@ -84,7 +83,7 @@ class WandbLogger():
|
|
84 |
self.job_type = job_type
|
85 |
self.wandb, self.wandb_run, self.data_dict = wandb, None if not wandb else wandb.run, data_dict
|
86 |
# It's more elegant to stick to 1 wandb.init call, but useful config data is overwritten in the WandbLogger's wandb.init call
|
87 |
-
if isinstance(opt.resume, str):
|
88 |
if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):
|
89 |
run_id, project, model_artifact_name = get_run_info(opt.resume)
|
90 |
model_artifact_name = WANDB_ARTIFACT_PREFIX + model_artifact_name
|
@@ -98,7 +97,7 @@ class WandbLogger():
|
|
98 |
project='YOLOv5' if opt.project == 'runs/train' else Path(opt.project).stem,
|
99 |
name=name,
|
100 |
job_type=job_type,
|
101 |
-
id=run_id) if not wandb.run else wandb.run
|
102 |
if self.wandb_run:
|
103 |
if self.job_type == 'Training':
|
104 |
if not opt.resume:
|
@@ -110,15 +109,15 @@ class WandbLogger():
|
|
110 |
if self.job_type == 'Dataset Creation':
|
111 |
self.data_dict = self.check_and_upload_dataset(opt)
|
112 |
else:
|
113 |
-
|
114 |
-
|
115 |
|
116 |
def check_and_upload_dataset(self, opt):
|
117 |
assert wandb, 'Install wandb to upload dataset'
|
118 |
check_dataset(self.data_dict)
|
119 |
config_path = self.log_dataset_artifact(opt.data,
|
120 |
-
|
121 |
-
|
122 |
print("Created dataset config file ", config_path)
|
123 |
with open(config_path) as f:
|
124 |
wandb_data_dict = yaml.load(f, Loader=yaml.SafeLoader)
|
|
|
|
|
1 |
import json
|
|
|
|
|
2 |
import sys
|
3 |
+
from pathlib import Path
|
4 |
+
|
5 |
import torch
|
6 |
import yaml
|
|
|
|
|
7 |
from tqdm import tqdm
|
8 |
|
9 |
sys.path.append(str(Path(__file__).parent.parent.parent)) # add utils/ to path
|
|
|
30 |
return wandb_config
|
31 |
return data_config_file
|
32 |
|
33 |
+
|
34 |
def get_run_info(run_path):
|
35 |
run_path = Path(remove_prefix(run_path, WANDB_ARTIFACT_PREFIX))
|
36 |
run_id = run_path.stem
|
|
|
38 |
model_artifact_name = 'run_' + run_id + '_model'
|
39 |
return run_id, project, model_artifact_name
|
40 |
|
41 |
+
|
42 |
def check_wandb_resume(opt):
|
43 |
process_wandb_config_ddp_mode(opt) if opt.global_rank not in [-1, 0] else None
|
44 |
if isinstance(opt.resume, str):
|
45 |
if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):
|
46 |
+
if opt.global_rank not in [-1, 0]: # For resuming DDP runs
|
47 |
run_id, project, model_artifact_name = get_run_info(opt.resume)
|
48 |
api = wandb.Api()
|
49 |
artifact = api.artifact(project + '/' + model_artifact_name + ':latest')
|
|
|
52 |
return True
|
53 |
return None
|
54 |
|
55 |
+
|
56 |
def process_wandb_config_ddp_mode(opt):
|
57 |
with open(opt.data) as f:
|
58 |
data_dict = yaml.load(f, Loader=yaml.SafeLoader) # data dict
|
|
|
63 |
train_dir = train_artifact.download()
|
64 |
train_path = Path(train_dir) / 'data/images/'
|
65 |
data_dict['train'] = str(train_path)
|
66 |
+
|
67 |
if isinstance(data_dict['val'], str) and data_dict['val'].startswith(WANDB_ARTIFACT_PREFIX):
|
68 |
api = wandb.Api()
|
69 |
val_artifact = api.artifact(remove_prefix(data_dict['val']) + ':' + opt.artifact_alias)
|
|
|
71 |
val_path = Path(val_dir) / 'data/images/'
|
72 |
data_dict['val'] = str(val_path)
|
73 |
if train_dir or val_dir:
|
74 |
+
ddp_data_path = str(Path(val_dir) / 'wandb_local_data.yaml')
|
75 |
with open(ddp_data_path, 'w') as f:
|
76 |
yaml.dump(data_dict, f)
|
77 |
opt.data = ddp_data_path
|
78 |
+
|
|
|
79 |
|
80 |
class WandbLogger():
|
81 |
def __init__(self, opt, name, run_id, data_dict, job_type='Training'):
|
|
|
83 |
self.job_type = job_type
|
84 |
self.wandb, self.wandb_run, self.data_dict = wandb, None if not wandb else wandb.run, data_dict
|
85 |
# It's more elegant to stick to 1 wandb.init call, but useful config data is overwritten in the WandbLogger's wandb.init call
|
86 |
+
if isinstance(opt.resume, str): # checks resume from artifact
|
87 |
if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):
|
88 |
run_id, project, model_artifact_name = get_run_info(opt.resume)
|
89 |
model_artifact_name = WANDB_ARTIFACT_PREFIX + model_artifact_name
|
|
|
97 |
project='YOLOv5' if opt.project == 'runs/train' else Path(opt.project).stem,
|
98 |
name=name,
|
99 |
job_type=job_type,
|
100 |
+
id=run_id) if not wandb.run else wandb.run
|
101 |
if self.wandb_run:
|
102 |
if self.job_type == 'Training':
|
103 |
if not opt.resume:
|
|
|
109 |
if self.job_type == 'Dataset Creation':
|
110 |
self.data_dict = self.check_and_upload_dataset(opt)
|
111 |
else:
|
112 |
+
prefix = colorstr('wandb: ')
|
113 |
+
print(f"{prefix}Install Weights & Biases for YOLOv5 logging with 'pip install wandb' (recommended)")
|
114 |
|
115 |
def check_and_upload_dataset(self, opt):
|
116 |
assert wandb, 'Install wandb to upload dataset'
|
117 |
check_dataset(self.data_dict)
|
118 |
config_path = self.log_dataset_artifact(opt.data,
|
119 |
+
opt.single_cls,
|
120 |
+
'YOLOv5' if opt.project == 'runs/train' else Path(opt.project).stem)
|
121 |
print("Created dataset config file ", config_path)
|
122 |
with open(config_path) as f:
|
123 |
wandb_data_dict = yaml.load(f, Loader=yaml.SafeLoader)
|