Spaces:
Running
Running
Add evaluation code
Browse files- evaluate.py +24 -35
evaluate.py
CHANGED
@@ -63,9 +63,25 @@ def get_solution_code(day: int, model: str) -> str:
|
|
63 |
def extract_solutions(df, output_file = "solutions.json"):
|
64 |
# TODO: better way of getting this?
|
65 |
solutions = {}
|
66 |
-
for day in range(1,
|
67 |
sub_df = df[(df.model == "jerpint") & (df.day == day)]
|
68 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
69 |
solutions[day] = [part1, part2]
|
70 |
|
71 |
with open(output_file, "w") as f:
|
@@ -82,8 +98,7 @@ def evaluate_submissions(all_models, results_file = "results.csv", skip = True):
|
|
82 |
else:
|
83 |
df = pd.DataFrame(columns=["day", "model", "result", "total_time"])
|
84 |
|
85 |
-
|
86 |
-
for day in range(1, 11):
|
87 |
print("*" * 80)
|
88 |
print(f"Evaluating day {day}")
|
89 |
for provider in all_models:
|
@@ -98,9 +113,13 @@ def evaluate_submissions(all_models, results_file = "results.csv", skip = True):
|
|
98 |
result = evaluate_submission(day, model)
|
99 |
df = pd.concat([df, pd.DataFrame({"day": [day], "model": [model], "result": [result["result"]], "total_time": [result["total_time"]]})], ignore_index=True)
|
100 |
|
|
|
101 |
df.to_csv("results.csv", index=False)
|
102 |
print("-" * 80)
|
|
|
103 |
print("*" * 80)
|
|
|
|
|
104 |
return df
|
105 |
|
106 |
|
@@ -109,35 +128,5 @@ if __name__ == "__main__":
|
|
109 |
all_models["human"] = ["jerpint"]
|
110 |
df = evaluate_submissions(all_models, results_file="results.csv")
|
111 |
|
112 |
-
# For now, only evaluate first 9 days
|
113 |
-
# TODO: All days
|
114 |
-
df = df[df.day < 10]
|
115 |
-
|
116 |
# Run once to save results
|
117 |
-
|
118 |
-
|
119 |
-
with open("solutions.json") as f:
|
120 |
-
solutions = json.load(f)
|
121 |
-
|
122 |
-
def score_submissions(row):
|
123 |
-
result = row["result"]
|
124 |
-
day = row["day"]
|
125 |
-
solution = solutions[str(day)]
|
126 |
-
|
127 |
-
score_1 = solution[0] in result
|
128 |
-
score_2 = solution[1] in result
|
129 |
-
return [score_1, score_2]
|
130 |
-
|
131 |
-
|
132 |
-
df["scores"] = df.apply(score_submissions, axis=1)
|
133 |
-
|
134 |
-
df["part_1"] = df["scores"].apply(lambda x: x[0])
|
135 |
-
df["part_2"] = df["scores"].apply(lambda x: x[1])
|
136 |
-
|
137 |
-
for model in df.model.unique():
|
138 |
-
df_model = df[df.model == model]
|
139 |
-
silver_stars = df_model.part_1.sum()
|
140 |
-
gold_stars = df_model.part_2.sum()
|
141 |
-
total_stars = silver_stars + gold_stars
|
142 |
-
|
143 |
-
print(model, total_stars)
|
|
|
63 |
def extract_solutions(df, output_file = "solutions.json"):
|
64 |
# TODO: better way of getting this?
|
65 |
solutions = {}
|
66 |
+
for day in range(1, 26):
|
67 |
sub_df = df[(df.model == "jerpint") & (df.day == day)]
|
68 |
+
|
69 |
+
|
70 |
+
day_solution = sub_df.result.to_list()[0].strip("\n").split("\n")
|
71 |
+
if len(day_solution) == 0:
|
72 |
+
part1 = "N/A"
|
73 |
+
part2 = "N/A"
|
74 |
+
elif len(day_solution) == 1:
|
75 |
+
part1 = day_solution[0]
|
76 |
+
part2 = "N/A"
|
77 |
+
elif len(day_solution) == 2:
|
78 |
+
part1, part2 = day_solution
|
79 |
+
else:
|
80 |
+
print("Something went wrong, check day {day} solution: \n {day_solution}")
|
81 |
+
part1 = "N/A"
|
82 |
+
part2 = "N/A"
|
83 |
+
|
84 |
+
|
85 |
solutions[day] = [part1, part2]
|
86 |
|
87 |
with open(output_file, "w") as f:
|
|
|
98 |
else:
|
99 |
df = pd.DataFrame(columns=["day", "model", "result", "total_time"])
|
100 |
|
101 |
+
for day in range(1, 26):
|
|
|
102 |
print("*" * 80)
|
103 |
print(f"Evaluating day {day}")
|
104 |
for provider in all_models:
|
|
|
113 |
result = evaluate_submission(day, model)
|
114 |
df = pd.concat([df, pd.DataFrame({"day": [day], "model": [model], "result": [result["result"]], "total_time": [result["total_time"]]})], ignore_index=True)
|
115 |
|
116 |
+
# Save incrementally
|
117 |
df.to_csv("results.csv", index=False)
|
118 |
print("-" * 80)
|
119 |
+
|
120 |
print("*" * 80)
|
121 |
+
df = df.sort_values(by="day")
|
122 |
+
df.to_csv("results.csv", index=False)
|
123 |
return df
|
124 |
|
125 |
|
|
|
128 |
all_models["human"] = ["jerpint"]
|
129 |
df = evaluate_submissions(all_models, results_file="results.csv")
|
130 |
|
|
|
|
|
|
|
|
|
131 |
# Run once to save results
|
132 |
+
solutions = extract_solutions(df, output_file="solutions.json")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|