File size: 4,575 Bytes
ff66cf3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 |
import os
import sys
import json
from cliport import agents
from cliport import tasks
import argparse
import datetime
import matplotlib as mpl
mpl.use("Agg")
import argparse
import os
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib
import IPython
import numpy as np
font = {
"size": 22,
}
matplotlib.rc("font", **font)
sns.set_context("paper", font_scale=2.0)
def mkdir_if_missing(dst_dir):
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
def save_figure(name, title=""):
print(f"output/output_figures/{name}.png")
if len(title) > 0:
plt.title(title)
plt.tight_layout()
mkdir_if_missing(f"output/output_figures/{name}")
plt.savefig(f"output/output_figures/{name}/output.png")
plt.clf()
def print_and_write(file_handle, text):
print(text)
if file_handle is not None:
file_handle.write(text + "\n")
return text
parser = argparse.ArgumentParser()
# federated arguments (Notation for the arguments followed from paper)
parser.add_argument(
"--results", "-r", type=str, default="exps/exps-singletask"
)
parser.add_argument(
"--single", "-s", action="store_true", default=False
)
args = parser.parse_args()
root_folder = os.environ['GENSIM_ROOT']
exp_folder = os.path.join(root_folder, args.results) # replace 'cliport_quickstart' with your exps folder
mkdir_if_missing('output/output_figures')
mkdir_if_missing('output/cliport_output')
mkdir_if_missing('output/output_stat')
output_stat_file = os.path.join('output/', 'cliport_output/', 'cliport-training.txt')
mkdir_if_missing('output/cliport_output/')
file_handle = open(output_stat_file, 'a+')
tasks_list = list(tasks.names.keys())
agents_list = list(agents.names.keys())
demos_list = [1, 5, 10, 20, 30, 50, 100, 200, 1000] # 100,
results = {}
for t in tasks_list:
for a in agents_list:
for d in demos_list:
task_folder = f'{t}-{a}-n{d}-train'
task_folder_path = os.path.join(exp_folder, task_folder, 'checkpoints')
if os.path.exists(task_folder_path):
print(f"train {task_folder_path}")
jsons = [f for f in os.listdir(task_folder_path) if '.json' in f]
for j in jsons:
model_type = 'multi' if 'multi' in j else 'single'
eval_type = 'val' if 'val' in j else 'test'
with open(os.path.join(task_folder_path, j)) as f:
res = json.load(f)
results[f'{t}-{a}-n{d}-{model_type}-{eval_type}'] = res
dt_string = datetime.datetime.now().strftime("%d_%m_%Y_%H:%M:%S")
print_and_write(file_handle, f"==========================={dt_string}=========================\n")
print_and_write(file_handle, f'Experiments folder: {exp_folder}\n')
data = {'task': [], 'success': []}
for eval_type in ['val', 'test']:
print_and_write(file_handle, f'----- {eval_type.upper()} -----\n')
for t in tasks_list:
for a in agents_list:
for d in demos_list:
for model_type in ['single', 'multi']:
eval_key = f'{t}-{a}-n{d}-{model_type}-{eval_type}'
if eval_key in results:
print_and_write(file_handle, f'{eval_key} {t} | Train Demos: {d}')
res = results[eval_key]
best_score, best_ckpt = max(zip([v['mean_reward'] for v in list(res.values())], res.keys()))
# TODO: test that this works for full results folder
print_and_write(file_handle, f'\t{best_score*100:1.1f} : {a} | {model_type}\n')
data['task'].append(t)
data['success'].append(best_score)
data['task'].append("Average")
data['success'].append(np.mean(data["success"]))
# make figure as well for sinle expeirment results
dfs = []
suffix = ""
run_num = 0
df = pd.DataFrame.from_dict(data)
title = args.results + "_res"
# rewards
fig, ax = plt.subplots(figsize=(16, 8))
sns_plot = sns.barplot(
data=df, x="task", y="success", errorbar=("sd", 1), palette="deep"
)
# label texts
for container in ax.containers:
ax.bar_label(container, label_type="center", fontsize="x-large", fmt="%.2f")
# ax.set_xticklabels(ax.get_xticklabels(), rotation=90, ha="right")
ax.set_xticklabels(['\n'.join(str(xlabel.get_text()).split("-")) for xlabel in ax.get_xticklabels()])
# save plot
save_figure(f"{title}", title)
|