test

README.md

@@ -9,8 +9,6 @@ app_file: app.py
 hf_oauth: true
 hf_oauth_scopes:
   - read-repos
-  - write-repos
-  - manage-repos
 pinned: false
 license: apache-2.0
 ---

app.py

@@ -11,7 +11,7 @@ from config_store import (
 )
 
 import gradio as gr
-from huggingface_hub import create_repo, whoami
+from huggingface_hub import whoami, login, logout
 from gradio_huggingfacehub_search import HuggingfaceHubSearch
 from optimum_benchmark.launchers.device_isolation_utils import *  # noqa
 from optimum_benchmark.backends.openvino.utils import TASKS_TO_OVMODEL
@@ -19,8 +19,8 @@ from optimum_benchmark.backends.transformers_utils import TASKS_TO_MODEL_LOADERS
 from optimum_benchmark import (
     Benchmark,
     BenchmarkConfig,
-    ProcessConfig,
     InferenceConfig,
+    ProcessConfig,
     PyTorchConfig,
     OVConfig,
 )
@@ -32,24 +32,12 @@ DEVICE = "cpu"
 LAUNCHER = "process"
 SCENARIO = "inference"
 BACKENDS = ["pytorch", "openvino"]
+BENCHMARKS_HF_TOKEN = os.getenv("BENCHMARKS_HF_TOKEN")
+BENCHMARKS_REPO_ID = "optimum-benchmark/OpenVINO-Benchmarks"
 TASKS = set(TASKS_TO_OVMODEL.keys()) & set(TASKS_TO_MODEL_LOADERS.keys())
 
 
-def run_benchmark(inputs, oauth_token: Optional[gr.OAuthToken] = None):
-    if oauth_token.token is None or oauth_token.token == "":
-        raise gr.Error("Please login to be able to run the benchmark.")
-
-    timestamp = time.strftime("%Y-%m-%d-%H-%M-%S")
-    name = whoami(oauth_token.token)["name"]
-    repo_id = f"{name}/benchmarks"
-    token = oauth_token.token
-
-    try:
-        create_repo(repo_id, token=token, repo_type="dataset", exist_ok=True)
-        gr.Info(f"📂 Created dataset repository {repo_id} on the Hub.")
-    except Exception:
-        raise gr.Error(f"Error while creating dataset repository {repo_id} on the Hub.")
-
+def parse_configs(inputs):
     configs = {"process": {}, "inference": {}, "pytorch": {}, "openvino": {}}
 
     for key, value in inputs.items():
@@ -77,46 +65,55 @@ def run_benchmark(inputs, oauth_token: Optional[gr.OAuthToken] = None):
         task=task, model=model, device=DEVICE, **configs["openvino"]
     )
 
-    outputs = {
-        "pytorch": "Running benchmark for PyTorch backend",
-        "openvino": "Running benchmark for OpenVINO backend",
-    }
+    return configs
 
+
+def run_benchmark(inputs, oauth_token: Optional[gr.OAuthToken]):
+    if oauth_token.token is None or oauth_token.token == "":
+        raise gr.Error("Please login to be able to run the benchmark.")
+
+    timestamp = time.strftime("%Y-%m-%d-%H-%M-%S")
+    use_name = whoami(oauth_token.token)["name"]
+    folder = f"{use_name}/{timestamp}"
+
+    gr.Info(f"📩 Benchmark will be saved under {BENCHMARKS_REPO_ID}/{folder}")
+
+    outputs = {backend: "Running..." for backend in BACKENDS}
     yield tuple(outputs[b] for b in BACKENDS)
+    configs = parse_configs(inputs)
 
     for backend in BACKENDS:
         try:
-            benchmark_name = f"{timestamp}/{backend}"
+            login(token=oauth_token.token)
+            benchmark_name = f"{folder}/{backend}"
             benchmark_config = BenchmarkConfig(
                 name=benchmark_name,
                 backend=configs[backend],
                 launcher=configs[LAUNCHER],
                 scenario=configs[SCENARIO],
             )
+            benchmark_report = Benchmark.launch(benchmark_config)
+            logout()
+
             benchmark_config.push_to_hub(
-                repo_id=repo_id, subfolder=benchmark_name, token=oauth_token.token
+                repo_id=BENCHMARKS_REPO_ID,
+                subfolder=benchmark_name,
+                token=BENCHMARKS_HF_TOKEN,
             )
-            benchmark_report = Benchmark.launch(benchmark_config)
             benchmark_report.push_to_hub(
-                repo_id=repo_id, subfolder=benchmark_name, token=oauth_token.token
-            )
-            benchmark = Benchmark(config=benchmark_config, report=benchmark_report)
-            benchmark.push_to_hub(
-                repo_id=repo_id, subfolder=benchmark_name, token=oauth_token.token
+                repo_id=BENCHMARKS_REPO_ID,
+                subfolder=benchmark_name,
+                token=BENCHMARKS_HF_TOKEN,
             )
 
         except Exception:
             outputs[backend] = f"\n```python-traceback\n{traceback.format_exc()}```\n"
-
             yield tuple(outputs[b] for b in BACKENDS)
-
             gr.Info(f"❌ Error while running benchmark for {backend} backend.")
 
         else:
             outputs[backend] = f"\n{benchmark_report.to_markdown_text()}\n"
-
             yield tuple(outputs[b] for b in BACKENDS)
-
             gr.Info(f"✅ Benchmark for {backend} backend ran successfully.")
 
 

requirements.txt

@@ -1,2 +1,2 @@
 gradio_huggingfacehub_search
-optimum-benchmark[openvino]@git+https://github.com/huggingface/optimum-benchmark.git@markdown-report
+optimum-benchmark[openvino]