File size: 1,887 Bytes
d023e59
 
ccb86e6
 
 
 
 
616b8af
d5b1458
 
81672b4
807b73e
216eab8
 
 
 
9c8006c
216eab8
 
9714a8a
216eab8
 
0351a9f
 
216eab8
 
 
 
 
 
 
 
 
 
 
 
3226c9d
216eab8
 
 
 
6c04bde
 
51ccf18
df1004b
ca32ce8
72e682c
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
#!/bin/bash

# TODO: I think this flag is related to getting fail logs; check.
set -e

export SPACE="EnergyStarAI/launch-computation-example"

echo "Not checking h100 -- already know it's not there."
#python /check_h100.py
echo "Attempting to run."
#if [[ $? = 0 ]]; then

# For each line in the requests dataset....
python /parse_requests.py | while read -r line; do
    # Read the name of the model and the experiment.
    IFS="," read backend_model experiment_name <<< "${line}"
    echo "Benchmarking Model: ${backend_model}, Task: ${experiment_name}"

    # Initialize the directory for output.
    now=$(date +%Y-%m-%d-%H-%M-%S)
    run_dir="./runs/${experiment_name}/${backend_model}/${now}"
    mkdir -p "$run_dir"

    # Let the benchmarking begin!
    optimum-benchmark --config-name "${experiment_name}"  --config-dir /optimum-benchmark/examples/energy_star/ backend.model="${backend_model}" backend.processor="${backend_model}" hydra.run.dir="${run_dir}" 2> "${run_dir}/error.log"

    # Either mark that the benchmark FAILED, or upload the results.
    if [ -s "${run_dir}/error.log" ]; then
      # error.log is not-empty, an error was raised
      echo "An error was raised while benchmarking the model..."
      python /failed_run.py --run_dir "${run_dir}" --model_name "${backend_model}"
    else
      # The error log file is empty, and we didn't catch an error.
      echo "Finished; uploading dataset results"
      python /create_results.py ./runs
    fi
done || {
  # Catch any errors that get thrown; update the requests dataset to FAILED
  # based on the last-read run_dir and backend_model.
  echo "Error."
  python /failed_run.py --run_dir "${run_dir}" --model_name "${backend_model}"
}

echo "Uploading all output from the /runs folder."
python /upload_run_folder.py --run_dir "/runs"

# Pausing space
echo "Pausing space."
python /pause_space.py
echo "Done."