glenn-jocher commited on
Commit
0cae757
·
unverified ·
1 Parent(s): b5de52c

utils/wandb_logging PEP8 reformat (#2755)

Browse files

* wandb_logging PEP8 reformat

* Update wandb_utils.py

utils/wandb_logging/log_dataset.py CHANGED
@@ -1,10 +1,8 @@
1
  import argparse
2
- from pathlib import Path
3
 
4
  import yaml
5
 
6
  from wandb_utils import WandbLogger
7
- from utils.datasets import LoadImagesAndLabels
8
 
9
  WANDB_ARTIFACT_PREFIX = 'wandb-artifact://'
10
 
@@ -21,6 +19,6 @@ if __name__ == '__main__':
21
  parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset')
22
  parser.add_argument('--project', type=str, default='YOLOv5', help='name of W&B Project')
23
  opt = parser.parse_args()
24
- opt.resume = False # Explicitly disallow resume check for dataset upload Job
25
-
26
  create_dataset_artifact(opt)
 
1
  import argparse
 
2
 
3
  import yaml
4
 
5
  from wandb_utils import WandbLogger
 
6
 
7
  WANDB_ARTIFACT_PREFIX = 'wandb-artifact://'
8
 
 
19
  parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset')
20
  parser.add_argument('--project', type=str, default='YOLOv5', help='name of W&B Project')
21
  opt = parser.parse_args()
22
+ opt.resume = False # Explicitly disallow resume check for dataset upload job
23
+
24
  create_dataset_artifact(opt)
utils/wandb_logging/wandb_utils.py CHANGED
@@ -1,12 +1,9 @@
1
- import argparse
2
  import json
3
- import os
4
- import shutil
5
  import sys
 
 
6
  import torch
7
  import yaml
8
- from datetime import datetime
9
- from pathlib import Path
10
  from tqdm import tqdm
11
 
12
  sys.path.append(str(Path(__file__).parent.parent.parent)) # add utils/ to path
@@ -33,6 +30,7 @@ def check_wandb_config_file(data_config_file):
33
  return wandb_config
34
  return data_config_file
35
 
 
36
  def get_run_info(run_path):
37
  run_path = Path(remove_prefix(run_path, WANDB_ARTIFACT_PREFIX))
38
  run_id = run_path.stem
@@ -40,11 +38,12 @@ def get_run_info(run_path):
40
  model_artifact_name = 'run_' + run_id + '_model'
41
  return run_id, project, model_artifact_name
42
 
 
43
  def check_wandb_resume(opt):
44
  process_wandb_config_ddp_mode(opt) if opt.global_rank not in [-1, 0] else None
45
  if isinstance(opt.resume, str):
46
  if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):
47
- if opt.global_rank not in [-1, 0]: # For resuming DDP runs
48
  run_id, project, model_artifact_name = get_run_info(opt.resume)
49
  api = wandb.Api()
50
  artifact = api.artifact(project + '/' + model_artifact_name + ':latest')
@@ -53,6 +52,7 @@ def check_wandb_resume(opt):
53
  return True
54
  return None
55
 
 
56
  def process_wandb_config_ddp_mode(opt):
57
  with open(opt.data) as f:
58
  data_dict = yaml.load(f, Loader=yaml.SafeLoader) # data dict
@@ -63,7 +63,7 @@ def process_wandb_config_ddp_mode(opt):
63
  train_dir = train_artifact.download()
64
  train_path = Path(train_dir) / 'data/images/'
65
  data_dict['train'] = str(train_path)
66
-
67
  if isinstance(data_dict['val'], str) and data_dict['val'].startswith(WANDB_ARTIFACT_PREFIX):
68
  api = wandb.Api()
69
  val_artifact = api.artifact(remove_prefix(data_dict['val']) + ':' + opt.artifact_alias)
@@ -71,12 +71,11 @@ def process_wandb_config_ddp_mode(opt):
71
  val_path = Path(val_dir) / 'data/images/'
72
  data_dict['val'] = str(val_path)
73
  if train_dir or val_dir:
74
- ddp_data_path = str(Path(val_dir) / 'wandb_local_data.yaml')
75
  with open(ddp_data_path, 'w') as f:
76
  yaml.dump(data_dict, f)
77
  opt.data = ddp_data_path
78
-
79
-
80
 
81
  class WandbLogger():
82
  def __init__(self, opt, name, run_id, data_dict, job_type='Training'):
@@ -84,7 +83,7 @@ class WandbLogger():
84
  self.job_type = job_type
85
  self.wandb, self.wandb_run, self.data_dict = wandb, None if not wandb else wandb.run, data_dict
86
  # It's more elegant to stick to 1 wandb.init call, but useful config data is overwritten in the WandbLogger's wandb.init call
87
- if isinstance(opt.resume, str): # checks resume from artifact
88
  if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):
89
  run_id, project, model_artifact_name = get_run_info(opt.resume)
90
  model_artifact_name = WANDB_ARTIFACT_PREFIX + model_artifact_name
@@ -98,7 +97,7 @@ class WandbLogger():
98
  project='YOLOv5' if opt.project == 'runs/train' else Path(opt.project).stem,
99
  name=name,
100
  job_type=job_type,
101
- id=run_id) if not wandb.run else wandb.run
102
  if self.wandb_run:
103
  if self.job_type == 'Training':
104
  if not opt.resume:
@@ -110,15 +109,15 @@ class WandbLogger():
110
  if self.job_type == 'Dataset Creation':
111
  self.data_dict = self.check_and_upload_dataset(opt)
112
  else:
113
- print(f"{colorstr('wandb: ')}Install Weights & Biases for YOLOv5 logging with 'pip install wandb' (recommended)")
114
-
115
 
116
  def check_and_upload_dataset(self, opt):
117
  assert wandb, 'Install wandb to upload dataset'
118
  check_dataset(self.data_dict)
119
  config_path = self.log_dataset_artifact(opt.data,
120
- opt.single_cls,
121
- 'YOLOv5' if opt.project == 'runs/train' else Path(opt.project).stem)
122
  print("Created dataset config file ", config_path)
123
  with open(config_path) as f:
124
  wandb_data_dict = yaml.load(f, Loader=yaml.SafeLoader)
 
 
1
  import json
 
 
2
  import sys
3
+ from pathlib import Path
4
+
5
  import torch
6
  import yaml
 
 
7
  from tqdm import tqdm
8
 
9
  sys.path.append(str(Path(__file__).parent.parent.parent)) # add utils/ to path
 
30
  return wandb_config
31
  return data_config_file
32
 
33
+
34
  def get_run_info(run_path):
35
  run_path = Path(remove_prefix(run_path, WANDB_ARTIFACT_PREFIX))
36
  run_id = run_path.stem
 
38
  model_artifact_name = 'run_' + run_id + '_model'
39
  return run_id, project, model_artifact_name
40
 
41
+
42
  def check_wandb_resume(opt):
43
  process_wandb_config_ddp_mode(opt) if opt.global_rank not in [-1, 0] else None
44
  if isinstance(opt.resume, str):
45
  if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):
46
+ if opt.global_rank not in [-1, 0]: # For resuming DDP runs
47
  run_id, project, model_artifact_name = get_run_info(opt.resume)
48
  api = wandb.Api()
49
  artifact = api.artifact(project + '/' + model_artifact_name + ':latest')
 
52
  return True
53
  return None
54
 
55
+
56
  def process_wandb_config_ddp_mode(opt):
57
  with open(opt.data) as f:
58
  data_dict = yaml.load(f, Loader=yaml.SafeLoader) # data dict
 
63
  train_dir = train_artifact.download()
64
  train_path = Path(train_dir) / 'data/images/'
65
  data_dict['train'] = str(train_path)
66
+
67
  if isinstance(data_dict['val'], str) and data_dict['val'].startswith(WANDB_ARTIFACT_PREFIX):
68
  api = wandb.Api()
69
  val_artifact = api.artifact(remove_prefix(data_dict['val']) + ':' + opt.artifact_alias)
 
71
  val_path = Path(val_dir) / 'data/images/'
72
  data_dict['val'] = str(val_path)
73
  if train_dir or val_dir:
74
+ ddp_data_path = str(Path(val_dir) / 'wandb_local_data.yaml')
75
  with open(ddp_data_path, 'w') as f:
76
  yaml.dump(data_dict, f)
77
  opt.data = ddp_data_path
78
+
 
79
 
80
  class WandbLogger():
81
  def __init__(self, opt, name, run_id, data_dict, job_type='Training'):
 
83
  self.job_type = job_type
84
  self.wandb, self.wandb_run, self.data_dict = wandb, None if not wandb else wandb.run, data_dict
85
  # It's more elegant to stick to 1 wandb.init call, but useful config data is overwritten in the WandbLogger's wandb.init call
86
+ if isinstance(opt.resume, str): # checks resume from artifact
87
  if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):
88
  run_id, project, model_artifact_name = get_run_info(opt.resume)
89
  model_artifact_name = WANDB_ARTIFACT_PREFIX + model_artifact_name
 
97
  project='YOLOv5' if opt.project == 'runs/train' else Path(opt.project).stem,
98
  name=name,
99
  job_type=job_type,
100
+ id=run_id) if not wandb.run else wandb.run
101
  if self.wandb_run:
102
  if self.job_type == 'Training':
103
  if not opt.resume:
 
109
  if self.job_type == 'Dataset Creation':
110
  self.data_dict = self.check_and_upload_dataset(opt)
111
  else:
112
+ prefix = colorstr('wandb: ')
113
+ print(f"{prefix}Install Weights & Biases for YOLOv5 logging with 'pip install wandb' (recommended)")
114
 
115
  def check_and_upload_dataset(self, opt):
116
  assert wandb, 'Install wandb to upload dataset'
117
  check_dataset(self.data_dict)
118
  config_path = self.log_dataset_artifact(opt.data,
119
+ opt.single_cls,
120
+ 'YOLOv5' if opt.project == 'runs/train' else Path(opt.project).stem)
121
  print("Created dataset config file ", config_path)
122
  with open(config_path) as f:
123
  wandb_data_dict = yaml.load(f, Loader=yaml.SafeLoader)