bishmoy commited on
Commit
f3cf0e1
·
verified ·
1 Parent(s): 4fef4b7

updated lb metrics

Browse files
Files changed (1) hide show
  1. leaderboard.py +47 -3
leaderboard.py CHANGED
@@ -1,5 +1,5 @@
1
  from glob import glob
2
- from sklearn.metrics import accuracy_score, recall_score
3
  import os
4
  import pandas as pd
5
 
@@ -56,7 +56,7 @@ def get_duration_scores(df):
56
  lb = pd.DataFrame({"Sample": columns, "Num Samples": samples_tested, "Accuracy": acc_scores})
57
  return lb
58
 
59
- def get_algorithm_scores(df):
60
 
61
  columns = list(df[df.label != 'real'].algorithm.unique())
62
  samples_tested = []
@@ -75,8 +75,52 @@ def get_algorithm_scores(df):
75
  lb = pd.DataFrame({"Sample": columns, "Num Samples": samples_tested, "Recall": rec_scores})
76
  return lb
77
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
78
  def build_leaderboard(results_path = 'results'):
79
  full_df = get_merged_df(results_path)
80
  full_df_mapped = map_df(full_df)
81
- leaderboard = get_algorithm_scores(full_df_mapped)
82
  return leaderboard
 
1
  from glob import glob
2
+ from sklearn.metrics import accuracy_score, recall_score, f1_score
3
  import os
4
  import pandas as pd
5
 
 
56
  lb = pd.DataFrame({"Sample": columns, "Num Samples": samples_tested, "Accuracy": acc_scores})
57
  return lb
58
 
59
+ def get_algorithm_scores_v1(df):
60
 
61
  columns = list(df[df.label != 'real'].algorithm.unique())
62
  samples_tested = []
 
75
  lb = pd.DataFrame({"Sample": columns, "Num Samples": samples_tested, "Recall": rec_scores})
76
  return lb
77
 
78
+ def get_algorithm_scores_v2(df):
79
+
80
+ columns = list(df[df.label != 'real'].algorithm.unique())
81
+ columns2 = list(df[df.label != 'real'].label.unique())
82
+ samples_tested = []
83
+ acc_scores = []
84
+ tpr_scores = []
85
+ tnr_scores = [float('nan')]*(len(columns) + len(columns2))
86
+ f1_scores = [float('nan')]*(len(columns) + len(columns2))
87
+
88
+ for c in columns:
89
+ mask = (df.algorithm == c)
90
+ sel_df = df[mask]
91
+
92
+ samples_tested.append(len(sel_df))
93
+ tpr_scores.append(round(recall_score(sel_df.gnd_truth.values, sel_df.pred.values, pos_label=1), 3))
94
+
95
+
96
+ for c in columns2:
97
+ mask = (df.label == c)
98
+ sel_df = df[mask]
99
+
100
+ samples_tested.append(len(sel_df))
101
+ tpr_scores.append(round(recall_score(sel_df.gnd_truth.values, sel_df.pred.values, pos_label=1), 3))
102
+
103
+ mask = (df.label != "real")
104
+ sel_df = df[mask]
105
+
106
+ tpr_scores.append(round(recall_score(sel_df.gnd_truth.values, sel_df.pred.values, pos_label=1), 3))
107
+
108
+ mask = (df.label == "real")
109
+ sel_df = df[mask]
110
+
111
+ tnr_scores.append(round(recall_score(sel_df.gnd_truth.values, sel_df.pred.values, pos_label=0), 3))
112
+
113
+ sel_df = df.copy()
114
+ samples_tested.append(len(sel_df))
115
+ f1_scores.append(round(f1_score(sel_df.gnd_truth.values, sel_df.pred.values, average="macro"), 3))
116
+
117
+
118
+ lb = pd.DataFrame({"Sample": columns + columns2 + ["overall (real + fake)"], "Num Samples": samples_tested,
119
+ "TPR": tpr_scores, "TNR": tnr_scores, "F1": f1_scores})
120
+ return lb
121
+
122
  def build_leaderboard(results_path = 'results'):
123
  full_df = get_merged_df(results_path)
124
  full_df_mapped = map_df(full_df)
125
+ leaderboard = get_algorithm_scores_v2(full_df_mapped)
126
  return leaderboard