failed_run.py

import argparse
import os

from datasets import load_dataset, Dataset
from huggingface_hub import HfApi


TOKEN = os.environ.get("DEBUG")
api = HfApi(token=TOKEN)

parser = argparse.ArgumentParser()
parser.add_argument(
    "--run_dir",
    default=None,
    type=str,
    required=True,
    help="Path to the run directory.",
)
parser.add_argument(
    "--model_name",
    default=None,
    type=str,
    required=True,
    help="Model to benchmark.",
)

args = parser.parse_args()

# Updating request
dataset = load_dataset("EnergyStarAI/requests_debug", split="test", token=TOKEN).to_pandas()

def upload_results():
    models_ran=[]
    for f in os.scandir('./runs'):
      if f.is_dir():
        for s in os.scandir(f):
          if s.is_dir() and s.name not in ['hooks','info','objects','refs','logs']:
              for m in os.scandir(s):
                models_ran.append(s.name+'/' + m.name)
                  
    print("Models ran are: " + str(models_ran))

requests_dset.loc[requests_dset["model"].isin(models_ran), ['status']] = "COMPLETED"
updated_dset =Dataset.from_pandas(requests_dset)
updated_dset.push_to_hub("EnergyStarAI/requests_debug", split="test", token=TOKEN)
print("Updated model status")

# Set benchmark to failed
# TODO: This doesn't have to be try-except, we could actually check if the file is there.
try:
    # Read error message
    with open(f"{args.run_dir}/error.log", 'r') as file:
        for f in file.readlines():
            if 'RuntimeError' in f:
                error_message = f
                dataset.loc[dataset["model"].isin([args.model_name]), ['status']] = "FAILED"
                print("Status set to FAILED")
            else:
                dataset.loc[dataset["model"].isin([args.model_name]), ['status']] = "COMPLETED"
                upload_results()
                print("Status set to COMPLETE")
    # Add a new column for the error message if necessary
    if "error_message" not in dataset.columns:
        dataset["error_message"] = ""
    dataset.loc[dataset["model"].isin([args.model_name]), ['error_message']] = error_message
except FileNotFoundError as e:
    print(f"Could not find {args.run_dir}/error.log")

updated_dataset = Dataset.from_pandas(dataset)
updated_dataset.push_to_hub("EnergyStarAI/requests_debug", split="test", token=TOKEN)