import requests import pandas as pd from tqdm.auto import tqdm import gradio as gr from huggingface_hub import HfApi, hf_hub_download from huggingface_hub.repocard import metadata_load # Based on Omar Sanseviero work # Make model clickable link def make_clickable_model(model_name): link = "https://huggingface.co/" + model_name return f'{model_name}' # Make user clickable link def make_clickable_user(user_id): link = "https://huggingface.co/" + user_id return f'{user_id}' def get_model_ids(rl_env): api = HfApi() models = api.list_models(filter=rl_env) model_ids = [x.modelId for x in models] return model_ids def get_metadata(model_id): try: readme_path = hf_hub_download(model_id, filename="README.md") return metadata_load(readme_path) except requests.exceptions.HTTPError: # 404 README.md not found return None def parse_metrics_accuracy(meta): if "model-index" not in meta: return None result = meta["model-index"][0]["results"] metrics = result[0]["metrics"] accuracy = metrics[0]["value"] return accuracy # We keep the worst case episode def parse_rewards(accuracy): if accuracy != None: parsed = accuracy.split(' +/- ') mean_reward = float(parsed[0]) std_reward = float(parsed[1]) else: mean_reward = -1000 std_reward = -1000 return mean_reward, std_reward def get_data(rl_env): data = [] model_ids = get_model_ids(rl_env) for model_id in tqdm(model_ids): meta = get_metadata(model_id) if meta is None: continue user_id = model_id.split('/')[0] row = {} row["User"] = user_id row["Model"] = model_id accuracy = parse_metrics_accuracy(meta) mean_reward, std_reward = parse_rewards(accuracy) row["Results"] = mean_reward - std_reward row["Mean Reward"] = mean_reward row["Std Reward"] = std_reward data.append(row) return pd.DataFrame.from_records(data) def get_data_per_env(rl_env): dataframe = get_data(rl_env) dataframe = dataframe.fillna("") if not dataframe.empty: # turn the model ids into clickable links dataframe["User"] = dataframe["User"].apply(make_clickable_user) dataframe["Model"] = dataframe["Model"].apply(make_clickable_model) dataframe = dataframe.sort_values(by=['Results'], ascending=False) table_html = dataframe.to_html(escape=False, index=False,justify = 'left') return table_html,dataframe,dataframe.empty else: html = """
⌛ Please wait. Results will be out soon...
This is a leaderboard of {len_dataframe} agents playing {env_name} 👩🚀.
We use lower bound result to sort the models: mean_reward - std_reward.
You can click on the model's name to be redirected to its model card which includes documentation.
You want to try your model? Read this Unit 1 of Deep Reinforcement Learning Class.