jerpint commited on
Commit
97a885e
·
1 Parent(s): 84520b3

Add evaluation code

Browse files
Files changed (1) hide show
  1. evaluate.py +24 -35
evaluate.py CHANGED
@@ -63,9 +63,25 @@ def get_solution_code(day: int, model: str) -> str:
63
  def extract_solutions(df, output_file = "solutions.json"):
64
  # TODO: better way of getting this?
65
  solutions = {}
66
- for day in range(1, 25):
67
  sub_df = df[(df.model == "jerpint") & (df.day == day)]
68
- part1, part2 = sub_df.result.to_list()[0].strip("\n").split("\n")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
69
  solutions[day] = [part1, part2]
70
 
71
  with open(output_file, "w") as f:
@@ -82,8 +98,7 @@ def evaluate_submissions(all_models, results_file = "results.csv", skip = True):
82
  else:
83
  df = pd.DataFrame(columns=["day", "model", "result", "total_time"])
84
 
85
- # for day in range(1, 26):
86
- for day in range(1, 11):
87
  print("*" * 80)
88
  print(f"Evaluating day {day}")
89
  for provider in all_models:
@@ -98,9 +113,13 @@ def evaluate_submissions(all_models, results_file = "results.csv", skip = True):
98
  result = evaluate_submission(day, model)
99
  df = pd.concat([df, pd.DataFrame({"day": [day], "model": [model], "result": [result["result"]], "total_time": [result["total_time"]]})], ignore_index=True)
100
 
 
101
  df.to_csv("results.csv", index=False)
102
  print("-" * 80)
 
103
  print("*" * 80)
 
 
104
  return df
105
 
106
 
@@ -109,35 +128,5 @@ if __name__ == "__main__":
109
  all_models["human"] = ["jerpint"]
110
  df = evaluate_submissions(all_models, results_file="results.csv")
111
 
112
- # For now, only evaluate first 9 days
113
- # TODO: All days
114
- df = df[df.day < 10]
115
-
116
  # Run once to save results
117
- # solutions = extract_solutions(df)
118
-
119
- with open("solutions.json") as f:
120
- solutions = json.load(f)
121
-
122
- def score_submissions(row):
123
- result = row["result"]
124
- day = row["day"]
125
- solution = solutions[str(day)]
126
-
127
- score_1 = solution[0] in result
128
- score_2 = solution[1] in result
129
- return [score_1, score_2]
130
-
131
-
132
- df["scores"] = df.apply(score_submissions, axis=1)
133
-
134
- df["part_1"] = df["scores"].apply(lambda x: x[0])
135
- df["part_2"] = df["scores"].apply(lambda x: x[1])
136
-
137
- for model in df.model.unique():
138
- df_model = df[df.model == model]
139
- silver_stars = df_model.part_1.sum()
140
- gold_stars = df_model.part_2.sum()
141
- total_stars = silver_stars + gold_stars
142
-
143
- print(model, total_stars)
 
63
  def extract_solutions(df, output_file = "solutions.json"):
64
  # TODO: better way of getting this?
65
  solutions = {}
66
+ for day in range(1, 26):
67
  sub_df = df[(df.model == "jerpint") & (df.day == day)]
68
+
69
+
70
+ day_solution = sub_df.result.to_list()[0].strip("\n").split("\n")
71
+ if len(day_solution) == 0:
72
+ part1 = "N/A"
73
+ part2 = "N/A"
74
+ elif len(day_solution) == 1:
75
+ part1 = day_solution[0]
76
+ part2 = "N/A"
77
+ elif len(day_solution) == 2:
78
+ part1, part2 = day_solution
79
+ else:
80
+ print("Something went wrong, check day {day} solution: \n {day_solution}")
81
+ part1 = "N/A"
82
+ part2 = "N/A"
83
+
84
+
85
  solutions[day] = [part1, part2]
86
 
87
  with open(output_file, "w") as f:
 
98
  else:
99
  df = pd.DataFrame(columns=["day", "model", "result", "total_time"])
100
 
101
+ for day in range(1, 26):
 
102
  print("*" * 80)
103
  print(f"Evaluating day {day}")
104
  for provider in all_models:
 
113
  result = evaluate_submission(day, model)
114
  df = pd.concat([df, pd.DataFrame({"day": [day], "model": [model], "result": [result["result"]], "total_time": [result["total_time"]]})], ignore_index=True)
115
 
116
+ # Save incrementally
117
  df.to_csv("results.csv", index=False)
118
  print("-" * 80)
119
+
120
  print("*" * 80)
121
+ df = df.sort_values(by="day")
122
+ df.to_csv("results.csv", index=False)
123
  return df
124
 
125
 
 
128
  all_models["human"] = ["jerpint"]
129
  df = evaluate_submissions(all_models, results_file="results.csv")
130
 
 
 
 
 
131
  # Run once to save results
132
+ solutions = extract_solutions(df, output_file="solutions.json")