Spaces:
Sleeping
Sleeping
File size: 6,397 Bytes
b78b0dc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 |
from peekingduck.pipeline.nodes.model import yolo as pkd_yolo
from src.data_ingestion.data_ingestion import AnnotsGTGetter
from src.inference import Inference
from src.confusion_matrix import ConfusionMatrix
import yaml
from itertools import product
import pandas as pd
def transform_gt_bbox_format(ground_truth, img_size, format = "coco"):
"""transforms ground truth bbox format to pascal voc for confusion matrix
Args:
ground_truth (_type_): nx5 numpy array, if coco - n x [class, x, y, w, h], if yolo - n x [class, x-mid, y-mid, w, h]
img_size (_type_): [Height * Weight * Dimension] values vector
format (str, optional): . Defaults to "coco".
Returns:
_type_: ground_truth. Transformed ground truth to pascal voc format
"""
if format == "coco":
ground_truth[:, 3] = (ground_truth[:, 1] + ground_truth[:, 3])/img_size[1]
ground_truth[:, 1] = (ground_truth[:, 1]) /img_size[1]
ground_truth[:, 4] = (ground_truth[:, 2] + ground_truth[:, 4])/img_size[0]
ground_truth[:, 2] = (ground_truth[:, 2]) /img_size[0]
return ground_truth
class ErrorAnalysis:
def __init__(self, cfg_path = 'cfg/cfg.yml'):
cfg_file = open(cfg_path)
self.cfg_obj = yaml.load(cfg_file, Loader=yaml.FullLoader)
# self.nms_thresh = self.cfg_obj['error_analysis']['nms_thresholds']
self.iou_thresh = self.cfg_obj['error_analysis']['iou_thresholds']
self.conf_thresh = self.cfg_obj['error_analysis']['conf_thresholds']
self.inference_folder = self.cfg_obj['dataset']['img_folder_path']
pkd = self.cfg_obj['error_analysis']['peekingduck']
self.cm_results = []
# todo - generalise the model
if pkd:
pkd_model = self.cfg_obj['pkd']['model']
# only instantiates the v4tiny model, but you are free to change this to other pkd model
if pkd_model == "yolo":
yolo_ver = self.cfg_obj['pkd']['yolo_ver']
self.model = pkd_yolo.Node(model_type = yolo_ver, detect= list(self.cfg_obj['error_analysis']['labels_dict'].keys()))
else:
# call in your own model
# self.model = <your model import here>
# make sure that your model has iou_threshold and score_threshold attributes
pass
def generate_inference(self, img_fname = "000000000139.jpg"):
"""Run inference on img based on the image file name. Path to the folder is determined by cfg
Args:
img_fname (str, optional): _description_. Defaults to "000000000139.jpg".
Returns:
ndarray, tuple: ndarray - n x [x1, y1, x2, y2, score, class], (H, W, D)
"""
inference_obj = Inference(self.model, self.cfg_obj)
img_path = f"{self.inference_folder}{img_fname}"
inference_outputs = inference_obj.run_inference_path(img_path)
return inference_outputs
def get_annots(self):
"""get GT annotations from dataset
"""
annots_obj = AnnotsGTGetter(cfg_obj = self.cfg_obj)
self.gt_dict = annots_obj.get_gt_annots()
def generate_conf_matrix(self,iou_threshold = 0.5, conf_threshold = 0.2):
"""generate the confusion matrix by running inference on each image
"""
num_classes = len(list(self.cfg_obj['error_analysis']['labels_dict'].keys()))
ground_truth_format = self.cfg_obj["error_analysis"]["ground_truth_format"]
idx_base = self.cfg_obj["error_analysis"]["idx_base"]
# TODO - currently, Conf Matrix is 0 indexed but all my classes are one-based index.
# need to find a better to resolve this
# Infuriating.
cm = ConfusionMatrix(num_classes=num_classes, CONF_THRESHOLD = conf_threshold, IOU_THRESHOLD=iou_threshold)
for fname in list(self.gt_dict.keys()):
inference_output, img_size = self.generate_inference(fname)
# deduct index_base from each inference's class index
inference_output[:, -1] -= idx_base
ground_truth = self.gt_dict[fname].copy()
# deduct index_base from each groundtruth's class index
ground_truth[:, 0] -= idx_base
# print (f"ground_truth: {ground_truth}")
# print (f"inference: {inference_output}")
# inference is in x1, y1, x2, y2, scores, class, so OK
# coco gt is in x, y, width, height - need to change to suit conf matrix
# img shape is (H, W, D) so plug in accordingly to normalise
ground_truth = transform_gt_bbox_format(ground_truth=ground_truth, img_size=img_size, format = ground_truth_format)
# print (f"ground_truth: {ground_truth}")
cm.process_batch(inference_output, ground_truth)
cm.get_PR()
return cm.matrix, cm.precision, cm.recall
def generate_conf_matrices(self, print_matrix = True):
"""generates the confidence matrices
"""
# get all combinations of the threshold values:
combinations = list(product(self.iou_thresh, self.conf_thresh))
# print (combinations)
comb_cms = {}
for comb in combinations:
# print (f"IOU: {comb[0]}, Conf: {comb[1]}")
self.model.iou_threshold, self.model.score_threshold = comb[0], comb[1]
returned_matrix, precision, recall = self.generate_conf_matrix(iou_threshold = comb[0], conf_threshold = comb[1])
# print (returned_matrix)
# print (f"precision: {precision}")
# print (f"recall: {recall}")
comb_cms[f"IOU: {comb[0]}, Conf: {comb[1]}"] = returned_matrix
self.cm_results.append([comb[0], comb[1], precision, recall])
if print_matrix:
for k, v in comb_cms.items():
print (k)
print (v)
def proc_pr_table(self):
self.cm_table = pd.DataFrame(self.cm_results, columns = ['IOU_Threshold', 'Score Threshold', 'Precision', 'Recall'])
print (self.cm_table)
if __name__ == "__main__":
ea_games = ErrorAnalysis()
# print (ea_games.generate_inference())
ea_games.get_annots()
ea_games.generate_conf_matrices()
# print (ea_games.generate_conf_matrix())
# print (ea_games.gt_dict) |