benchmark_threading.py

import os
import time
import argparse
import subprocess
import platform
from typing import Optional, Tuple, Dict
import threading
import numpy as np
from onnx import load, ModelProto
import onnxruntime as ort

os.environ["XLNX_ENABLE_CACHE"] = "0"
os.environ["PATH"] += (
    os.pathsep + f"{os.environ['CONDA_PREFIX']}\\Lib\\site-packages\\flexmlrt\\lib"
)

XRT_SMI_PATH = "C:\\Windows\\System32\\AMD\\xrt-smi.exe"

ONNX_DTYPE_TO_NP = {
    "tensor(float)": np.float32,
    "tensor(float16)": np.float16,
    "tensor(int64)": np.int64,
    "tensor(int32)": np.int32,
    "tensor(uint16)": np.uint16,
    "tensor(int16)": np.int16,
    "tensor(uint8)": np.uint8,
    "tensor(int8)": np.int8,
}


def generate_rand_data_from_onnx(
    model: ModelProto,
    lowest_int_val: Optional[int],
    highest_int_val: Optional[int],
) -> Dict[str, np.ndarray]:
    # Load the models

    sess = ort.InferenceSession(
        model.SerializePartialToString(), providers=["CPUExecutionProvider"]
    )

    inps = {}
    # Iterate over the first models inputs and generate random data
    for inp in sess.get_inputs():
        inp_shapes = list(inp.shape)  # mutable
        for inp_shape in inp_shapes:
            assert isinstance(
                inp_shape, int
            ), f"Found dynamic axes: {inp_shape}. Please freeze."
        np_type = ONNX_DTYPE_TO_NP[inp.type]
        if np.issubdtype(np_type, np.integer):
            iinfo = np.iinfo(np_type)
            if lowest_int_val is None:
                lowest_int_val = iinfo.min
            if highest_int_val is None:
                lowest_int_val = iinfo.max
            inps[inp.name] = np.random.randint(
                lowest_int_val, highest_int_val, size=tuple(inp_shapes), dtype=np_type
            )
        else:
            inps[inp.name] = np.random.rand(*inp_shapes).astype(np_type)

    return inps


def configure_npu_power(p_mode: Optional[str] = None) -> Tuple[int, str, str]:
    """
    Configures the NPU power state using xrt-smi.exe.

    Args:
        p_mode (string, optional): The desired power mode (p-mode).
            If None, displays current status.
            Refer to xrt-smi documentation for valid p-modes.
    Returns:
        tuple: (return_code, stdout, stderr) from the subprocess call.
               return_code is an integer, stdout and stderr are strings.
    Raises:
        OSError: If xrt-smi.exe is not found.
    """

    if platform.system() != "Windows":
        return (-1, "xrt-smi.exe is only available on Windows.", "")

    try:
        if p_mode is not None:
            command = [XRT_SMI_PATH, "configure", "--pmode", str(p_mode)]
        else:
            command = [
                XRT_SMI_PATH,
                "examine",
                "--report",
                "platform",
            ]  # Just display status

        process = subprocess.Popen(
            command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
        )
        stdout, stderr = process.communicate()
        return_code = process.returncode

        if return_code != 0:
            print(f"Error executing xrt-smi.exe: {stderr}")

        return return_code, stdout, stderr

    except FileNotFoundError:
        raise OSError("xrt-smi.exe not found.")
    except Exception as e:  # pylint: disable=broad-except
        print(f"An unexpected error occurred: {e}")
        return -1, "", str(e)


def main(
    model_file: str,
    vaip_config: str,
    cache_path: str,
    device: str,
    pmode: str,
    warmup_runs: int,
    inferences: int,
    lowest_int_value: Optional[int],
    highest_int_value: Optional[int],
    threads: int,
):

    assert os.path.exists(model_file)

    assert threads >= 1

    if device == "cpu":
        ort_session = ort.InferenceSession(
            model_file,
            providers=["CPUExecutionProvider"],
        )

    elif device == "npu":
        assert os.path.exists(vaip_config)
        assert os.path.exists(cache_path)
        cache_dir = os.path.dirname(os.path.abspath(cache_path))
        cache_key = os.path.basename(cache_path)
        print(f"Using cache directory {cache_dir} with key {cache_key}")

        ret_code, stdout, stderr = configure_npu_power(pmode)
        print(stdout)
        if ret_code != 0:
            print("Error configuring npu power mode.")
            print(stderr)

        sess_options = ort.SessionOptions()
        ort_session = ort.InferenceSession(
            model_file,
            providers=["VitisAIExecutionProvider"],
            sess_options=sess_options,
            provider_options=[
                {
                    "config_file": vaip_config,
                    "cacheDir": cache_dir,
                    "cacheKey": cache_key,
                }
            ],
        )

    elif device == "igpu":
        ort_session = ort.InferenceSession(
            model_file,
            providers=["DmlExecutionProvider"],
            provider_options=[{"device_id": 2}],
        )

    onnx_inputs = generate_rand_data_from_onnx(
        load(model_file),
        lowest_int_val=lowest_int_value,
        highest_int_val=highest_int_value,
    )

    # Warmup
    for _ in range(warmup_runs):
        ort_session.run(None, onnx_inputs)

    def run_inference(runs, session, inputs):
        for _ in range(runs):
            session.run(None, inputs)

    latencies = []
    num_threads = threads
    threads_list = []
    inferences_per_thread = inferences // num_threads
    remainder = inferences % num_threads
    print(f"inferences per thread: {inferences_per_thread}, remainder: {remainder}")
    start = time.perf_counter()
    for i in range(num_threads):
        num_runs = inferences_per_thread + (1 if i < remainder else 0)
        thread = threading.Thread(
            target=run_inference, args=(num_runs, ort_session, onnx_inputs)
        )
        threads_list.append(thread)
        thread.start()

    for thread in threads_list:
        thread.join()

    end = time.perf_counter()
    latencies.append((end - start) / inferences)
    print(f"Latencies: {latencies}")
    print(f"Benchmark results averaged over {inferences} inferences targeting {device}")
    print("Average latency (ms): ", round(np.mean(latencies) * 1e3, 3))
    print("Average throughput (inf/s): ", round(1 / np.mean(latencies), 3))


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="",
    )

    parser.add_argument(
        "--pmode",
        type=str,
        choices=["default", "powersaver", "balanced", "performance", "turbo"],
        default="default",
        help="Desired power mode.",
    )

    parser.add_argument(
        "onnx_model",
        type=str,
        help="Provide the onnx model file.",
    )

    parser.add_argument(
        "--vai-config",
        type=str,
        help="Path to the vaip configuration json file.",
    )

    parser.add_argument(
        "--cache-path",
        required=False,
        type=str,
        help="Path to the saved compilation directory.",
    )

    parser.add_argument(
        "--device",
        required=False,
        type=str,
        default="npu",
        choices=["npu", "cpu", "igpu"],
        help="Select the device to run the measurements on.",
    )

    parser.add_argument(
        "--warmup-runs",
        required=False,
        default=10,
        type=int,
        help="The number of inferences to run before capturing performance.",
    )

    parser.add_argument(
        "--inferences",
        required=False,
        default=100,
        type=int,
        help="The number of inferences to average performance over.",
    )

    parser.add_argument(
        "--lowest-int-value",
        required=False,
        type=int,
        help="Lowest value the rng will produce if the model has an integer input type.",
    )

    parser.add_argument(
        "--highest-int-value",
        required=False,
        type=int,
        help="Highest value the rng will produce if the model has an integer input type.",
    )

    parser.add_argument(
        "--threads",
        default=1,
        required=False,
        type=int,
        help="The number of threads that are used to run the inferences.",
    )

    args = parser.parse_args()

    main(
        args.onnx_model,
        args.vai_config,
        args.cache_path,
        args.device,
        args.pmode,
        args.warmup_runs,
        args.inferences,
        args.lowest_int_value,
        args.highest_int_value,
        args.threads,
    )