restructure

README.md

@@ -4,16 +4,9 @@ emoji: 🐢🔍
 colorFrom: blue
 colorTo: indigo
 sdk: gradio
-sdk_version: 4.16.0
+sdk_version: 4.7.1
 app_file: app.py
 pinned: false
-
-hf_oauth: true
-# optional, default duration is 8 hours/480 minutes. Max duration is 30 days/43200 minutes.
-hf_oauth_expiration_minutes: 480
-# optional, see "Scopes" below. "openid profile" is always included.
-hf_oauth_scopes:
- - inference-api
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -5,7 +5,7 @@ import gradio as gr
 from app_debug import get_demo as get_demo_debug
 from app_leaderboard import get_demo as get_demo_leaderboard
 from app_text_classification import get_demo as get_demo_text_classification
-from run_jobs import start_process_run_job, stop_thread
+from utils.run_jobs import start_process_run_job, stop_thread
 
 try:
     with gr.Blocks(theme=gr.themes.Soft(primary_hue="green")) as demo:

app_debug.py

@@ -3,12 +3,12 @@ from os.path import isfile, join
 import html
 
 import gradio as gr
-import os
-import pipe
-from io_utils import get_logs_file
+
+import utils.pipe as pipe
+from utils.io_utils import get_logs_file
 
 LOG_PATH = "./tmp"
-CONFIG_PATH = "./cicd/configs/submitted/"
+CONFIG_PATH = "./cicd/configs/"
 MAX_FILES_NUM = 20
 
 
@@ -69,19 +69,17 @@ def get_queue_status():
 
 
 def get_demo():
-    if not os.path.exists(CONFIG_PATH):
-        os.makedirs(CONFIG_PATH)
     with gr.Row():
         gr.HTML(        
             value=get_queue_status,
             every=5,
         )
-    with gr.Accordion(label="Log Files", open=True):
+    with gr.Accordion(label="Log Files", open=False):
+        with gr.Row():
+            gr.Files(value=get_log_files, label="Log Files", every=10)
         with gr.Row():
             gr.Textbox(
                 value=get_logs_file, every=0.5, lines=10, visible=True, label="Current Log File"
             )
-        with gr.Row():
-            gr.Files(value=get_log_files, label="Log Files", every=10)
     with gr.Accordion(label="Config Files", open=False):
         gr.Files(value=get_config_files, label="Config Files", every=10)

app_leaderboard.py

@@ -5,10 +5,10 @@ import gradio as gr
 import pandas as pd
 import datetime
 
-from fetch_utils import (check_dataset_and_get_config,
+from utils.fetch_utils import (check_dataset_and_get_config,
                          check_dataset_and_get_split)
 
-import leaderboard
+import utils.leaderboard as leaderboard
 logger = logging.getLogger(__name__)
 global update_time 
 update_time = datetime.datetime.fromtimestamp(0)
@@ -88,29 +88,11 @@ def get_demo(leaderboard_tab):
     dataset_ids = get_dataset_ids(records)
 
     column_names = records.columns.tolist()
-    issue_columns = column_names[:11]
-    info_columns = column_names[15:]
     default_columns = ["model_id", "dataset_id", "total_issues", "report_link"]
     default_df = records[default_columns]  # extract columns selected
     types = get_types(default_df)
     display_df = get_display_df(default_df)  # the styled dataframe to display
 
-    with gr.Row():
-        with gr.Column():
-          issue_columns_select = gr.CheckboxGroup(
-              label="Issue Columns",
-              choices=issue_columns,
-              value=[],
-              interactive=True,
-          )
-        with gr.Column():
-          info_columns_select = gr.CheckboxGroup(
-              label="Info Columns",
-              choices=info_columns,
-              value=default_columns,
-              interactive=True,
-          )
-    
     with gr.Row():
         task_select = gr.Dropdown(
             label="Task",
@@ -128,35 +110,42 @@ def get_demo(leaderboard_tab):
             interactive=True,
         )
 
+    with gr.Row():
+        columns_select = gr.CheckboxGroup(
+            label="Show columns",
+            choices=column_names,
+            value=default_columns,
+            interactive=True,
+        )
+
     with gr.Row():
         leaderboard_df = gr.DataFrame(display_df, datatype=types, interactive=False)
 
-    def update_leaderboard_records(model_id, dataset_id, issue_columns, info_columns, task):
+    def update_leaderboard_records(model_id, dataset_id, columns, task):
         global update_time
         if datetime.datetime.now() - update_time < datetime.timedelta(minutes=10):
             return gr.update()
         update_time = datetime.datetime.now()
         logger.info("Updating leaderboard records")
         leaderboard.records = get_records_from_dataset_repo(leaderboard.LEADERBOARD)
-        return filter_table(model_id, dataset_id, issue_columns, info_columns, task)
+        return filter_table(model_id, dataset_id, columns, task)
 
     leaderboard_tab.select(
         fn=update_leaderboard_records, 
-        inputs=[model_select, dataset_select, issue_columns_select, info_columns_select, task_select], 
+        inputs=[model_select, dataset_select, columns_select, task_select], 
         outputs=[leaderboard_df])
 
     @gr.on(
         triggers=[
             model_select.change,
             dataset_select.change,
-            issue_columns_select.change,
-            info_columns_select.change,
+            columns_select.change,
             task_select.change,
         ],
-        inputs=[model_select, dataset_select, issue_columns_select, info_columns_select, task_select],
+        inputs=[model_select, dataset_select, columns_select, task_select],
         outputs=[leaderboard_df],
     )
-    def filter_table(model_id, dataset_id, issue_columns, info_columns, task):
+    def filter_table(model_id, dataset_id, columns, task):
         logger.info("Filtering leaderboard records")
         records = leaderboard.records
         # filter the table based on task
@@ -167,9 +156,8 @@ def get_demo(leaderboard_tab):
         if dataset_id and dataset_id != "Any":
             df = df[(df["dataset_id"] == dataset_id)]
 
-        # filter the table based on the columns 
-        issue_columns.sort()
-        df = df[info_columns + issue_columns]
+        # filter the table based on the columns
+        df = df[columns]
         types = get_types(df)
         display_df = get_display_df(df)
         return gr.update(value=display_df, datatype=types, interactive=False)

app_legacy.py

@@ -376,7 +376,7 @@ def get_demo():
         selected = read_scanners("./config.yaml")
         scan_config = selected + ["data_leakage"]
         scanners = gr.CheckboxGroup(
-            choices=scan_config, value=selected, visible=True
+            choices=scan_config, value=selected, label="Scan Settings", visible=True
         )
 
     with gr.Row():

app_text_classification.py

@@ -2,12 +2,12 @@ import uuid
 
 import gradio as gr
 
-from io_utils import read_scanners, write_scanners
-from text_classification_ui_helpers import (
+from utils.io_utils import read_scanners, write_scanners
+from utils.ui_helpers import (
     get_related_datasets_from_leaderboard,
     align_columns_and_show_prediction,
-    get_dataset_splits,
     check_dataset,
+    show_hf_token_info,
     precheck_model_ds_enable_example_btn,
     try_submit,
     empty_column_mapping,
@@ -16,11 +16,12 @@ from text_classification_ui_helpers import (
 )
 
 import logging
-from wordings import (
+from utils.wordings import (
   CONFIRM_MAPPING_DETAILS_MD, 
   INTRODUCTION_MD, 
-  LOG_IN_TIPS, 
+  USE_INFERENCE_API_TIP, 
   CHECK_LOG_SECTION_RAW,
+  HF_TOKEN_INVALID_STYLED
 )
 
 MAX_LABELS = 40
@@ -33,16 +34,9 @@ logger = logging.getLogger(__name__)
 def get_demo():
     with gr.Row():
         gr.Markdown(INTRODUCTION_MD)
-
-    with gr.Row(visible=False):
         uid_label = gr.Textbox(
             label="Evaluation ID:", value=uuid.uuid4, visible=False, interactive=False
         )
-
-    with gr.Accordion(label="Log In", open=True):
-        gr.HTML(LOG_IN_TIPS)
-        gr.LoginButton()
-
     with gr.Row():
         model_id_input = gr.Textbox(
             label="Hugging Face Model id",
@@ -64,7 +58,7 @@ def get_demo():
     with gr.Row():
         first_line_ds = gr.DataFrame(label="Dataset Preview", visible=False)
     with gr.Row():
-        loading_dataset_info = gr.HTML(visible=True)
+        loading_status = gr.HTML(visible=True)
     with gr.Row():
         example_btn = gr.Button(
             "Validate Model & Dataset",
@@ -72,13 +66,11 @@ def get_demo():
             variant="primary",
             interactive=False,
         )
+
     with gr.Row():
-        loading_validation = gr.HTML(visible=True)
-    with gr.Row():
-        validation_result = gr.HTML(visible=False)
+        example_input = gr.HTML(visible=False)
     with gr.Row():
-        example_input = gr.Textbox(label="Example Input", visible=False, interactive=False)
-        example_prediction = gr.Label(label="Model Sample Prediction", visible=False)
+        example_prediction = gr.Label(label="Model Prediction Sample", visible=False)
 
     with gr.Row():
         with gr.Accordion(
@@ -97,8 +89,27 @@ def get_demo():
                     for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES):
                         column_mappings.append(gr.Dropdown(visible=False))
 
-    with gr.Accordion(label="Scanner Advanced Config (optional)", open=False):
-        scanners = gr.CheckboxGroup(visible=True)
+    with gr.Accordion(label="Model Wrap Advance Config", open=True):
+        gr.HTML(USE_INFERENCE_API_TIP)
+
+        run_inference = gr.Checkbox(value=True, label="Run with Inference API")
+        inference_token = gr.Textbox(
+            placeholder="hf-xxxxxxxxxxxxxxxxxxxx",
+            value="",
+            label="HF Token for Inference API",
+            visible=True,
+            interactive=True,
+        )
+        inference_token_info = gr.HTML(value=HF_TOKEN_INVALID_STYLED, visible=False)
+
+        inference_token.change(
+            fn=show_hf_token_info,
+            inputs=[inference_token],
+            outputs=[inference_token_info],
+        )
+
+    with gr.Accordion(label="Scanner Advance Config (optional)", open=False):
+        scanners = gr.CheckboxGroup(label="Scan Settings", visible=True)
 
         @gr.on(triggers=[uid_label.change], inputs=[uid_label], outputs=[scanners])
         def get_scanners(uid):
@@ -106,16 +117,7 @@ def get_demo():
             # we remove data_leakage from the default scanners
             # Reason: data_leakage barely raises any issues and takes too many requests
             # when using inference API, causing rate limit error
-            scan_config = [
-                "ethical_bias", 
-                "text_perturbation", 
-                "robustness",
-                "performance",
-                "underconfidence",
-                "overconfidence",
-                "spurious_correlation",
-                "data_leakage",
-                ]
+            scan_config = selected + ["data_leakage"]
             return gr.update(
                 choices=scan_config, value=selected, label="Scan Settings", visible=True
             )
@@ -145,20 +147,18 @@ def get_demo():
         inputs=[model_id_input],
         outputs=[dataset_id_input],
     ).then(
-        fn=check_dataset,
-        inputs=[dataset_id_input],
-        outputs=[dataset_config_input, dataset_split_input, loading_dataset_info],
+        fn=check_dataset, 
+        inputs=[dataset_id_input], 
+        outputs=[dataset_config_input, dataset_split_input, loading_status]
     )
     
     gr.on(
-        triggers=[dataset_id_input.input, dataset_id_input.select],
+        triggers=[dataset_id_input.change],
         fn=check_dataset,
         inputs=[dataset_id_input],
-        outputs=[dataset_config_input, dataset_split_input, loading_dataset_info]
+        outputs=[dataset_config_input, dataset_split_input, loading_status]
     )
 
-    dataset_config_input.change(fn=get_dataset_splits, inputs=[dataset_id_input, dataset_config_input], outputs=[dataset_split_input])
-
     gr.on(
         triggers=[model_id_input.change, dataset_id_input.change, dataset_config_input.change],
         fn=empty_column_mapping,
@@ -187,7 +187,6 @@ def get_demo():
     gr.on(
         triggers=[
             model_id_input.change,
-            model_id_input.input,
             dataset_id_input.change,
             dataset_config_input.change,
             dataset_split_input.change,
@@ -199,13 +198,7 @@ def get_demo():
             dataset_config_input,
             dataset_split_input,
         ],
-        outputs=[
-            example_btn, 
-            first_line_ds,
-            validation_result,
-            example_input,
-            example_prediction,
-            column_mapping_accordion,],
+        outputs=[example_btn, first_line_ds, loading_status],
     )
 
     gr.on(
@@ -219,14 +212,15 @@ def get_demo():
             dataset_config_input,
             dataset_split_input,
             uid_label,
+            run_inference,
+            inference_token,
         ],
         outputs=[
-            validation_result,
             example_input,
             example_prediction,
             column_mapping_accordion,
             run_btn,
-            loading_validation,
+            loading_status,
             *column_mappings,
         ],
     )
@@ -241,26 +235,24 @@ def get_demo():
             dataset_id_input,
             dataset_config_input,
             dataset_split_input,
+            run_inference,
+            inference_token,
             uid_label,
         ],
-        outputs=[
-            run_btn, 
-            logs, 
-            uid_label,            
-            validation_result,
-            example_input,
-            example_prediction,
-            column_mapping_accordion,
-          ],
+        outputs=[run_btn, logs, uid_label],
     )
 
     gr.on(
         triggers=[
+            run_inference.input,
+            inference_token.input,
             scanners.input,
         ],
         fn=enable_run_btn,
         inputs=[
             uid_label,
+            run_inference, 
+            inference_token, 
             model_id_input, 
             dataset_id_input, 
             dataset_config_input, 
@@ -274,6 +266,8 @@ def get_demo():
         fn=enable_run_btn,
         inputs=[
             uid_label,
+            run_inference, 
+            inference_token, 
             model_id_input, 
             dataset_id_input, 
             dataset_config_input, 

fetch_utils.py

@@ -1,32 +0,0 @@
-import logging
-
-import datasets
-
-
-def check_dataset_and_get_config(dataset_id):
-    try:
-        configs = datasets.get_dataset_config_names(dataset_id, trust_remote_code=True)
-        return configs
-    except Exception:
-        # Dataset may not exist
-        return None
-
-
-def check_dataset_and_get_split(dataset_id, dataset_config):
-    try:
-        ds = datasets.load_dataset(dataset_id, dataset_config, trust_remote_code=True)
-    except Exception as e:
-        # Dataset may not exist
-        logging.warning(
-            f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}"
-        )
-        return None
-    try:
-        splits = list(ds.keys())
-        return splits
-    except Exception as e:
-        # Dataset has no splits
-        logging.warning(
-            f"Dataset {dataset_id} with config {dataset_config} has no splits: {e}"
-        )
-        return None

io_utils.py

@@ -1,134 +0,0 @@
-import os
-import logging
-import yaml
-
-YAML_PATH = "./cicd/configs"
-LOG_FILE = "temp_log"
-
-logger = logging.getLogger(__name__)
-
-class Dumper(yaml.Dumper):
-    def increase_indent(self, flow=False, *args, **kwargs):
-        return super().increase_indent(flow=flow, indentless=False)
-
-def get_submitted_yaml_path(uid):
-    if not os.path.exists(f"{YAML_PATH}/submitted"):
-        os.makedirs(f"{YAML_PATH}/submitted")
-    if not os.path.exists(f"{YAML_PATH}/{uid}_config.yaml"):
-        logger.error(f"config.yaml does not exist for {uid}")
-        os.system(f"cp config.yaml {YAML_PATH}/{uid}_config.yaml")
-    if not os.path.exists(f"{YAML_PATH}/submitted/{uid}_config.yaml"):
-        os.system(f"cp {YAML_PATH}/{uid}_config.yaml {YAML_PATH}/submitted/{uid}_config.yaml")
-    return f"{YAML_PATH}/submitted/{uid}_config.yaml"
-
-def get_yaml_path(uid):
-    if not os.path.exists(YAML_PATH):
-        os.makedirs(YAML_PATH)
-    if not os.path.exists(f"{YAML_PATH}/{uid}_config.yaml"):
-        os.system(f"cp config.yaml {YAML_PATH}/{uid}_config.yaml")
-    return f"{YAML_PATH}/{uid}_config.yaml"
-
-
-# read scanners from yaml file
-# return a list of scanners
-def read_scanners(uid):
-    scanners = []
-    with open(get_yaml_path(uid), "r") as f:
-        config = yaml.load(f, Loader=yaml.FullLoader)
-        scanners = config.get("detectors", [])
-    return scanners
-
-
-# convert a list of scanners to yaml file
-def write_scanners(scanners, uid):
-    with open(get_yaml_path(uid), "r") as f:
-        config = yaml.load(f, Loader=yaml.FullLoader)
-        if config:
-            config["detectors"] = scanners
-    # save scanners to detectors in yaml
-    with open(get_yaml_path(uid), "w") as f:
-        yaml.dump(config, f, Dumper=Dumper)
-
-
-# read model_type from yaml file
-def read_inference_type(uid):
-    inference_type = ""
-    with open(get_yaml_path(uid), "r") as f:
-        config = yaml.load(f, Loader=yaml.FullLoader)
-        inference_type = config.get("inference_type", "")
-    return inference_type
-
-
-# write model_type to yaml file
-def write_inference_type(use_inference, inference_token, uid):
-    with open(get_yaml_path(uid), "r") as f:
-        config = yaml.load(f, Loader=yaml.FullLoader)
-        if use_inference:
-            config["inference_type"] = "hf_inference_api"
-            config["inference_token"] = inference_token
-        else:
-            config["inference_type"] = "hf_pipeline"
-            # FIXME: A quick and temp fix for missing token
-            config["inference_token"] = ""
-    # save inference_type to inference_type in yaml
-    with open(get_yaml_path(uid), "w") as f:
-        yaml.dump(config, f, Dumper=Dumper)
-
-
-# read column mapping from yaml file
-def read_column_mapping(uid):
-    column_mapping = {}
-    with open(get_yaml_path(uid), "r") as f:
-        config = yaml.load(f, Loader=yaml.FullLoader)
-        if config:
-            column_mapping = config.get("column_mapping", dict())
-    if column_mapping is None:
-        column_mapping = {}
-    return column_mapping
-
-
-# write column mapping to yaml file
-def write_column_mapping(mapping, uid):
-    with open(get_yaml_path(uid), "r") as f:
-        config = yaml.load(f, Loader=yaml.FullLoader)
-
-    if config is None:
-        return
-    if mapping is None and "column_mapping" in config.keys():
-        del config["column_mapping"]
-    else:
-        config["column_mapping"] = mapping
-    with open(get_yaml_path(uid), "w") as f:
-        # yaml Dumper will by default sort the keys
-        yaml.dump(config, f, Dumper=Dumper, sort_keys=False)
-
-
-# convert column mapping dataframe to json
-def convert_column_mapping_to_json(df, label=""):
-    column_mapping = {}
-    column_mapping[label] = []
-    for _, row in df.iterrows():
-        column_mapping[label].append(row.tolist())
-    return column_mapping
-
-
-def get_log_file_with_uid(uid):
-    try:
-        print(f"Loading {uid}.log")
-        with open(f"./tmp/{uid}.log", "a") as file:
-            return file.read()
-    except Exception:
-        return "Log file does not exist"
-
-
-def get_logs_file():
-    try:
-        with open(LOG_FILE, "r") as file:
-            return file.read()
-    except Exception:
-        return "Log file does not exist"
-
-
-def write_log_to_user_file(task_id, log):
-    with open(f"./tmp/{task_id}.log", "a") as f:
-        f.write(log)

isolated_env.py

@@ -1,7 +1,7 @@
 import os
 import subprocess
 
-from io_utils import write_log_to_user_file
+from utils.io_utils import write_log_to_user_file
 
 
 def prepare_venv(execution_id, deps):

leaderboard.py

@@ -1,5 +0,0 @@
-import pandas as pd
-
-records = pd.DataFrame()
-
-LEADERBOARD = "giskard-bot/evaluator-leaderboard"

pipe.py

@@ -1,3 +0,0 @@
-
-jobs = list()
-current = None

requirements.txt

@@ -4,6 +4,4 @@ hf-transfer
 torch==2.0.1
 transformers
 datasets
-tabulate
-gradio[oauth]==4.19.2
 -e git+https://github.com/Giskard-AI/cicd.git#egg=giskard-cicd

run_jobs.py

@@ -1,181 +0,0 @@
-import json
-import logging
-import os
-import subprocess
-import threading
-import time
-from pathlib import Path
-
-import pipe
-from app_env import (
-    HF_GSK_HUB_HF_TOKEN,
-    HF_GSK_HUB_KEY,
-    HF_GSK_HUB_PROJECT_KEY,
-    HF_GSK_HUB_UNLOCK_TOKEN,
-    HF_GSK_HUB_URL,
-    HF_REPO_ID,
-    HF_SPACE_ID,
-    HF_WRITE_TOKEN,
-)
-from io_utils import LOG_FILE, get_submitted_yaml_path, write_log_to_user_file
-from isolated_env import prepare_venv
-from leaderboard import LEADERBOARD
-
-is_running = False
-
-logger = logging.getLogger(__file__)
-
-
-def start_process_run_job():
-    try:
-        logging.debug("Running jobs in thread")
-        global thread, is_running
-        thread = threading.Thread(target=run_job)
-        thread.daemon = True
-        is_running = True
-        thread.start()
-
-    except Exception as e:
-        print("Failed to start thread: ", e)
-
-
-def stop_thread():
-    logging.debug("Stop thread")
-    global is_running
-    is_running = False
-
-
-def prepare_env_and_get_command(
-    m_id,
-    d_id,
-    config,
-    split,
-    inference_token,
-    uid,
-    label_mapping,
-    feature_mapping,
-):
-    leaderboard_dataset = None
-    if os.environ.get("SPACE_ID") == "giskardai/giskard-evaluator":
-        leaderboard_dataset = LEADERBOARD
-
-    executable = "giskard_scanner"
-    try:
-        # Copy the current requirements (might be changed)
-        with open("requirements.txt", "r") as f:
-            executable = prepare_venv(
-                uid,
-                "\n".join(f.readlines()),
-            )
-        logger.info(f"Using {executable} as executable")
-    except Exception as e:
-        logger.warn(f"Create env failed due to {e}, using the current env as fallback.")
-        executable = "giskard_scanner"
-
-    command = [
-        executable,
-        "--loader",
-        "huggingface",
-        "--model",
-        m_id,
-        "--dataset",
-        d_id,
-        "--dataset_config",
-        config,
-        "--dataset_split",
-        split,
-        "--output_format",
-        "markdown",
-        "--output_portal",
-        "huggingface",
-        "--feature_mapping",
-        json.dumps(feature_mapping),
-        "--label_mapping",
-        json.dumps(label_mapping),
-        "--scan_config",
-        get_submitted_yaml_path(uid),
-        "--inference_type",
-        "hf_inference_api",
-        "--inference_api_token",
-        inference_token,
-    ]
-    # The token to publish post
-    if os.environ.get(HF_WRITE_TOKEN):
-        command.append("--hf_token")
-        command.append(os.environ.get(HF_WRITE_TOKEN))
-
-    # The repo to publish post
-    if os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID):
-        command.append("--discussion_repo")
-        # TODO: Replace by the model id
-        command.append(os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID))
-
-    # The repo to publish for ranking
-    if leaderboard_dataset:
-        command.append("--leaderboard_dataset")
-        command.append(leaderboard_dataset)
-
-    # The info to upload to Giskard hub
-    if os.environ.get(HF_GSK_HUB_KEY):
-        command.append("--giskard_hub_api_key")
-        command.append(os.environ.get(HF_GSK_HUB_KEY))
-        if os.environ.get(HF_GSK_HUB_URL):
-            command.append("--giskard_hub_url")
-            command.append(os.environ.get(HF_GSK_HUB_URL))
-        if os.environ.get(HF_GSK_HUB_PROJECT_KEY):
-            command.append("--giskard_hub_project_key")
-            command.append(os.environ.get(HF_GSK_HUB_PROJECT_KEY))
-        if os.environ.get(HF_GSK_HUB_HF_TOKEN):
-            command.append("--giskard_hub_hf_token")
-            command.append(os.environ.get(HF_GSK_HUB_HF_TOKEN))
-        if os.environ.get(HF_GSK_HUB_UNLOCK_TOKEN):
-            command.append("--giskard_hub_unlock_token")
-            command.append(os.environ.get(HF_GSK_HUB_UNLOCK_TOKEN))
-
-    eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
-
-    write_log_to_user_file(
-        uid,
-        f"Start local evaluation on {eval_str}. Please wait for your job to start...\n",
-    )
-
-    return command
-
-
-def save_job_to_pipe(task_id, job, description, lock):
-    with lock:
-        pipe.jobs.append((task_id, job, description))
-
-
-def pop_job_from_pipe():
-    if len(pipe.jobs) == 0:
-        return
-    job_info = pipe.jobs.pop()
-    pipe.current = job_info[2]
-    task_id = job_info[0]
-
-    # Link to LOG_FILE
-    log_file_path = Path(LOG_FILE)
-    if log_file_path.exists():
-        log_file_path.unlink()
-    os.symlink(f"./tmp/{task_id}.log", LOG_FILE)
-
-    write_log_to_user_file(task_id, f"Running job id {task_id}\n")
-    command = prepare_env_and_get_command(*job_info[1])
-
-    with open(f"./tmp/{task_id}.log", "a") as log_file:
-        p = subprocess.Popen(command, stdout=log_file, stderr=subprocess.STDOUT)
-        p.wait()
-    pipe.current = None
-
-
-def run_job():
-    global is_running
-    while is_running:
-        try:
-            pop_job_from_pipe()
-            time.sleep(10)
-        except KeyboardInterrupt:
-            logging.debug("KeyboardInterrupt stop background thread")
-            is_running = False
-            break

text_classification.py

@@ -1,409 +0,0 @@
-import json
-import logging
-
-import datasets
-import huggingface_hub
-import pandas as pd
-from transformers import pipeline
-import requests
-import os
-from app_env import HF_WRITE_TOKEN
-
-logger = logging.getLogger(__name__)
-AUTH_CHECK_URL = "https://huggingface.co/api/whoami-v2"
-
-logger = logging.getLogger(__file__)
-
-class HuggingFaceInferenceAPIResponse:
-    def __init__(self, message):
-        self.message = message
-
-
-def get_labels_and_features_from_dataset(ds):
-    try:
-        dataset_features = ds.features
-        label_keys = [i for i in dataset_features.keys() if i.startswith("label")]
-        features = [f for f in dataset_features.keys() if not f.startswith("label")]
-
-        if len(label_keys) == 0: # no labels found
-            # return everything for post processing
-            return list(dataset_features.keys()), list(dataset_features.keys()), None
-        
-        labels = None
-        if not isinstance(dataset_features[label_keys[0]], datasets.ClassLabel):
-            if hasattr(dataset_features[label_keys[0]], "feature"):
-                label_feat = dataset_features[label_keys[0]].feature
-                labels = label_feat.names
-            else:
-                labels = ds.unique(label_keys[0])
-        else:
-            labels = dataset_features[label_keys[0]].names
-        return labels, features, label_keys
-    except Exception as e:
-        logging.warning(
-            f"Get Labels/Features Failed for dataset: {e}"
-        )
-        return None, None, None
-
-def check_model_task(model_id):
-    # check if model is valid on huggingface
-    try:
-        task = huggingface_hub.model_info(model_id).pipeline_tag
-        if task is None:
-            return None
-        return task
-    except Exception:
-        return None
-
-def get_model_labels(model_id, example_input):
-    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
-    payload = {"inputs": example_input, "options": {"use_cache": True}}
-    response = hf_inference_api(model_id, hf_token, payload)
-    if "error" in response:
-        return None
-    return extract_from_response(response, "label")
-
-def extract_from_response(data, key):
-    results = []
-
-    if isinstance(data, dict):
-        res = data.get(key)
-        if res is not None:
-            results.append(res)
-
-        for value in data.values():
-            results.extend(extract_from_response(value, key))
-
-    elif isinstance(data, list):
-        for element in data:
-            results.extend(extract_from_response(element, key))
-
-    return results
-
-def hf_inference_api(model_id, hf_token, payload):
-    hf_inference_api_endpoint = os.environ.get(
-        "HF_INFERENCE_ENDPOINT", default="https://api-inference.huggingface.co"
-    )
-    url = f"{hf_inference_api_endpoint}/models/{model_id}"
-    headers = {"Authorization": f"Bearer {hf_token}"}
-    response = requests.post(url, headers=headers, json=payload)
-
-    if not hasattr(response, "status_code") or response.status_code != 200:
-        logger.warning(f"Request to inference API returns {response}")
-  
-    try:
-        output = response.json()
-        if "error" in output and "Input is too long" in output["error"]:
-          payload.update({"parameters": {"truncation": True, "max_length": 512}})
-          response = requests.post(url, headers=headers, json=payload)
-          if not hasattr(response, "status_code") or response.status_code != 200:
-              logger.warning(f"Request to inference API returns {response}")
-        return response.json()
-    except Exception:                
-        return {"error": response.content}
-    
-def preload_hf_inference_api(model_id):
-    payload = {"inputs": "This is a test", "options": {"use_cache": True, }}
-    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
-    hf_inference_api(model_id, hf_token, payload)
-
-def check_model_pipeline(model_id):
-    try:
-        task = huggingface_hub.model_info(model_id).pipeline_tag
-    except Exception:
-        return None
-
-    try:
-        ppl = pipeline(task=task, model=model_id)
-
-        return ppl
-    except Exception:
-        return None
-
-
-def text_classificaiton_match_label_case_unsensative(id2label_mapping, label):
-    for model_label in id2label_mapping.keys():
-        if model_label.upper() == label.upper():
-            return model_label, label
-    return None, label
-
-
-def text_classification_map_model_and_dataset_labels(id2label, dataset_features):
-    id2label_mapping = {id2label[k]: None for k in id2label.keys()}
-    dataset_labels = None
-    for feature in dataset_features.values():
-        if not isinstance(feature, datasets.ClassLabel):
-            continue
-        if len(feature.names) != len(id2label_mapping.keys()):
-            continue
-
-        dataset_labels = feature.names
-        # Try to match labels
-        for label in feature.names:
-            if label in id2label_mapping.keys():
-                model_label = label
-            else:
-                # Try to find case unsensative
-                model_label, label = text_classificaiton_match_label_case_unsensative(
-                    id2label_mapping, label
-                )
-            if model_label is not None:
-                id2label_mapping[model_label] = label
-            else:
-                print(f"Label {label} is not found in model labels")
-
-    return id2label_mapping, dataset_labels
-
-
-"""
-params:
-    column_mapping: dict
-    example: {
-        "text": "sentences",
-        "label": {
-            "label0": "LABEL_0",
-            "label1": "LABEL_1"
-        }
-    }
-    ppl: pipeline
-"""
-
-
-def check_column_mapping_keys_validity(column_mapping, ppl):
-    # get the element in all the list elements
-    column_mapping = json.loads(column_mapping)
-    if "data" not in column_mapping.keys():
-        return True
-    user_labels = set([pair[0] for pair in column_mapping["data"]])
-    model_labels = set([pair[1] for pair in column_mapping["data"]])
-
-    id2label = ppl.model.config.id2label
-    original_labels = set(id2label.values())
-
-    return user_labels == model_labels == original_labels
-
-
-"""
-params:
-    column_mapping: dict
-    dataset_features: dict
-    example: {
-        'text': Value(dtype='string', id=None), 
-        'label': ClassLabel(names=['negative', 'neutral', 'positive'], id=None)
-    }
-"""
-
-
-def infer_text_input_column(column_mapping, dataset_features):
-    # Check whether we need to infer the text input column
-    infer_text_input_column = True
-    feature_map_df = None
-
-    if "text" in column_mapping.keys():
-        dataset_text_column = column_mapping["text"]
-        if dataset_text_column in dataset_features.keys():
-            infer_text_input_column = False
-        else:
-            logging.warning(f"Provided {dataset_text_column} is not in Dataset columns")
-
-    if infer_text_input_column:
-        # Try to retrieve one
-        candidates = [
-            f for f in dataset_features if dataset_features[f].dtype == "string"
-        ]
-        feature_map_df = pd.DataFrame(
-            {"Dataset Features": [candidates[0]], "Model Input Features": ["text"]}
-        )
-        if len(candidates) > 0:
-            logging.debug(f"Candidates are {candidates}")
-            column_mapping["text"] = candidates[0]
-
-    return column_mapping, feature_map_df
-
-
-"""
-params:
-    column_mapping: dict
-    id2label_mapping: dict
-    example:
-    id2label_mapping: {
-        'negative': 'negative', 
-        'neutral': 'neutral', 
-        'positive': 'positive'
-        }
-"""
-
-
-def infer_output_label_column(
-    column_mapping, id2label_mapping, id2label, dataset_labels
-):
-    # Check whether we need to infer the output label column
-    if "data" in column_mapping.keys():
-        if isinstance(column_mapping["data"], list):
-            # Use the column mapping passed by user
-            for user_label, model_label in column_mapping["data"]:
-                id2label_mapping[model_label] = user_label
-    elif None in id2label_mapping.values():
-        column_mapping["label"] = {i: None for i in id2label.keys()}
-        return column_mapping, None
-
-    if "data" not in column_mapping.keys():
-        # Column mapping should contain original model labels
-        column_mapping["label"] = {
-            str(i): id2label_mapping[label]
-            for i, label in zip(id2label.keys(), dataset_labels)
-        }
-
-    id2label_df = pd.DataFrame(
-        {
-            "Dataset Labels": dataset_labels,
-            "Model Prediction Labels": [
-                id2label_mapping[label] for label in dataset_labels
-            ],
-        }
-    )
-
-    return column_mapping, id2label_df
-
-
-def check_dataset_features_validity(d_id, config, split):
-    # We assume dataset is ok here
-    ds = datasets.load_dataset(d_id, config, split=split, trust_remote_code=True)
-    try:
-        dataset_features = ds.features
-    except AttributeError:
-        # Dataset does not have features, need to provide everything
-        return None, None
-        # Load dataset as DataFrame
-    df = ds.to_pandas()
-
-    return df, dataset_features
-
-def select_the_first_string_column(ds):
-    for feature in ds.features.keys():
-        if isinstance(ds[0][feature], str):
-            return feature
-    return None
-
-
-def get_example_prediction(model_id, dataset_id, dataset_config, dataset_split, hf_token):
-    # get a sample prediction from the model on the dataset
-    prediction_input = None
-    prediction_result = None
-    try:
-        # Use the first item to test prediction
-        ds = datasets.load_dataset(dataset_id, dataset_config, split=dataset_split, trust_remote_code=True)
-        if "text" not in ds.features.keys():
-            # Dataset does not have text column
-            prediction_input = ds[0][select_the_first_string_column(ds)]
-        else:
-            prediction_input = ds[0]["text"]
-
-        payload = {"inputs": prediction_input, "options": {"use_cache": True}}
-        results = hf_inference_api(model_id, hf_token, payload)
-
-        if isinstance(results, dict) and "error" in results.keys():
-            if "estimated_time" in results.keys():
-                return prediction_input, HuggingFaceInferenceAPIResponse(
-                    f"Estimated time: {int(results['estimated_time'])}s. Please try again later.")
-            return prediction_input, HuggingFaceInferenceAPIResponse(
-                f"Inference Error: {results['error']}.")
-        
-        while isinstance(results, list):
-            if isinstance(results[0], dict):
-                break
-            results = results[0]
-        prediction_result = {
-            f'{result["label"]}': result["score"] for result in results
-        }
-    except Exception as e:
-        # inference api prediction failed, show the error message
-        logger.error(f"Get example prediction failed {e}")
-        return prediction_input, None
-
-    return prediction_input, prediction_result
-
-
-def get_sample_prediction(ppl, df, column_mapping, id2label_mapping):
-    # get a sample prediction from the model on the dataset
-    prediction_input = None
-    prediction_result = None
-    try:
-        # Use the first item to test prediction
-        prediction_input = df.head(1).at[0, column_mapping["text"]]
-        results = ppl({"text": prediction_input}, top_k=None)
-        prediction_result = {
-            f'{result["label"]}': result["score"] for result in results
-        }
-    except Exception:
-        # Pipeline prediction failed, need to provide labels
-        return prediction_input, None
-
-    # Display results in original label and mapped label
-    prediction_result = {
-        f'{result["label"]}(original) - {id2label_mapping[result["label"]]}(mapped)': result[
-            "score"
-        ]
-        for result in results
-    }
-    return prediction_input, prediction_result
-
-
-def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, split):
-    # load dataset as pd DataFrame
-    # get features column from dataset
-    df, dataset_features = check_dataset_features_validity(d_id, config, split)
-
-    column_mapping, feature_map_df = infer_text_input_column(
-        column_mapping, dataset_features
-    )
-    if feature_map_df is None:
-        # dataset does not have any features
-        return None, None, None, None, None
-
-    # Retrieve all labels
-    id2label = ppl.model.config.id2label
-
-    # Infer labels
-    id2label_mapping, dataset_labels = text_classification_map_model_and_dataset_labels(
-        id2label, dataset_features
-    )
-    column_mapping, id2label_df = infer_output_label_column(
-        column_mapping, id2label_mapping, id2label, dataset_labels
-    )
-    if id2label_df is None:
-        # does not able to infer output label column
-        return column_mapping, None, None, None, feature_map_df
-
-    # Get a sample prediction
-    prediction_input, prediction_result = get_sample_prediction(
-        ppl, df, column_mapping, id2label_mapping
-    )
-    if prediction_result is None:
-        # does not able to get a sample prediction
-        return column_mapping, prediction_input, None, id2label_df, feature_map_df
-
-    return (
-        column_mapping,
-        prediction_input,
-        prediction_result,
-        id2label_df,
-        feature_map_df,
-    )
-
-def strip_model_id_from_url(model_id):
-    if model_id.startswith("https://huggingface.co/"):
-        return "/".join(model_id.split("/")[-2:])
-    return model_id
-
-def check_hf_token_validity(hf_token):
-    if hf_token == "":
-        return False
-    if not isinstance(hf_token, str):
-        return False
-    # use huggingface api to check the token
-    headers = {"Authorization": f"Bearer {hf_token}"}
-    response = requests.get(AUTH_CHECK_URL, headers=headers)
-    if response.status_code != 200:
-        return False
-    return True

text_classification_ui_helpers.py

@@ -1,458 +0,0 @@
-import collections
-import logging
-import threading
-import uuid
-
-import datasets
-import gradio as gr
-import pandas as pd
-
-import leaderboard
-from io_utils import (
-  read_column_mapping, 
-  write_column_mapping, 
-  read_scanners, 
-  write_scanners,
-)
-from run_jobs import save_job_to_pipe
-from text_classification import (
-    strip_model_id_from_url,
-    check_model_task,
-    preload_hf_inference_api,
-    get_example_prediction,
-    get_labels_and_features_from_dataset,
-    check_hf_token_validity,
-    HuggingFaceInferenceAPIResponse,
-)
-from wordings import (
-    CHECK_CONFIG_OR_SPLIT_RAW,
-    CONFIRM_MAPPING_DETAILS_FAIL_RAW,
-    MAPPING_STYLED_ERROR_WARNING,
-    NOT_TEXT_CLASSIFICATION_MODEL_RAW,
-    UNMATCHED_MODEL_DATASET_STYLED_ERROR,
-    CHECK_LOG_SECTION_RAW,
-    VALIDATED_MODEL_DATASET_STYLED,
-    get_dataset_fetch_error_raw,
-)
-import os
-from app_env import HF_WRITE_TOKEN
-
-MAX_LABELS = 40
-MAX_FEATURES = 20
-
-ds_dict = None
-ds_config = None
-
-def get_related_datasets_from_leaderboard(model_id):
-    records = leaderboard.records
-    model_id = strip_model_id_from_url(model_id)
-    model_records = records[records["model_id"] == model_id]
-    datasets_unique = list(model_records["dataset_id"].unique())
-
-    if len(datasets_unique) == 0:
-        return gr.update(choices=[])
-    
-    return gr.update(choices=datasets_unique)
-
-
-logger = logging.getLogger(__file__)
-
-def get_dataset_splits(dataset_id, dataset_config):
-    try:
-        splits = datasets.get_dataset_split_names(dataset_id, dataset_config, trust_remote_code=True)
-        return gr.update(choices=splits, value=splits[0], visible=True)
-    except Exception as e:
-        logger.warn(f"Check your dataset {dataset_id} and config {dataset_config}: {e}")
-        return gr.update(visible=False)
-
-def check_dataset(dataset_id):
-    logger.info(f"Loading {dataset_id}")
-    try:
-        configs = datasets.get_dataset_config_names(dataset_id, trust_remote_code=True)
-        if len(configs) == 0:
-            return (
-                gr.update(visible=False),
-                gr.update(visible=False),
-                ""
-            )
-        splits = datasets.get_dataset_split_names(dataset_id, configs[0], trust_remote_code=True)
-        return (
-            gr.update(choices=configs, value=configs[0], visible=True),
-            gr.update(choices=splits, value=splits[0], visible=True),
-            ""
-        )
-    except Exception as e:
-        logger.warn(f"Check your dataset {dataset_id}: {e}")
-        if "doesn't exist" in str(e):
-            gr.Warning(get_dataset_fetch_error_raw(e))
-        if "forbidden" in str(e).lower(): # GSK-2770
-            gr.Warning(get_dataset_fetch_error_raw(e))
-        return (
-            gr.update(visible=False),
-            gr.update(visible=False),
-            ""
-        )
-
-def empty_column_mapping(uid):
-    write_column_mapping(None, uid)
-
-def write_column_mapping_to_config(uid, *labels):
-    # TODO: Substitute 'text' with more features for zero-shot
-    # we are not using ds features because we only support "text" for now
-    all_mappings = read_column_mapping(uid)
-
-    if labels is None:
-        return
-    all_mappings = export_mappings(all_mappings, "labels", None, labels[:MAX_LABELS])
-    all_mappings = export_mappings(
-        all_mappings,
-        "features",
-        ["text"],
-        labels[MAX_LABELS : (MAX_LABELS + MAX_FEATURES)],
-    )
-
-    write_column_mapping(all_mappings, uid)
-
-def export_mappings(all_mappings, key, subkeys, values):
-    if key not in all_mappings.keys():
-        all_mappings[key] = dict()
-    if subkeys is None:
-        subkeys = list(all_mappings[key].keys())
-
-    if not subkeys: 
-        logging.debug(f"subkeys is empty for {key}")
-        return all_mappings
-
-    for i, subkey in enumerate(subkeys):
-        if subkey:
-            all_mappings[key][subkey] = values[i % len(values)]
-    return all_mappings
-
-
-def list_labels_and_features_from_dataset(ds_labels, ds_features, model_labels, uid):
-    all_mappings = read_column_mapping(uid)
-    # For flattened raw datasets with no labels
-    # check if there are shared labels between model and dataset
-    shared_labels = set(model_labels).intersection(set(ds_labels))
-    if shared_labels:
-        ds_labels = list(shared_labels)
-    if len(ds_labels) > MAX_LABELS:
-        ds_labels = ds_labels[:MAX_LABELS]
-        gr.Warning(f"Too many labels to display for this spcae. We do not support more than {MAX_LABELS} in this space. You can use cli tool at https://github.com/Giskard-AI/cicd.")
-
-    # sort labels to make sure the order is consistent
-    # prediction gives the order based on probability
-    ds_labels.sort()
-    model_labels.sort()
-
-    lables = [
-        gr.Dropdown(
-            label=f"{label}",
-            choices=model_labels,
-            value=model_labels[i % len(model_labels)],
-            interactive=True,
-            visible=True,
-        )
-        for i, label in enumerate(ds_labels)
-    ]
-    lables += [gr.Dropdown(visible=False) for _ in range(MAX_LABELS - len(lables))]
-    all_mappings = export_mappings(all_mappings, "labels", ds_labels, model_labels)
-
-    # TODO: Substitute 'text' with more features for zero-shot
-    features = [
-        gr.Dropdown(
-            label=f"{feature}",
-            choices=ds_features,
-            value=ds_features[0],
-            interactive=True,
-            visible=True,
-        )
-        for feature in ["text"]
-    ]
-    features += [
-        gr.Dropdown(visible=False) for _ in range(MAX_FEATURES - len(features))
-    ]
-    all_mappings = export_mappings(all_mappings, "features", ["text"], ds_features)
-    write_column_mapping(all_mappings, uid)
-
-    return lables + features
-
-
-def precheck_model_ds_enable_example_btn(
-    model_id, dataset_id, dataset_config, dataset_split
-):  
-    model_id = strip_model_id_from_url(model_id)
-    model_task = check_model_task(model_id)
-    preload_hf_inference_api(model_id)
-
-    if dataset_config is None or dataset_split is None or len(dataset_config) == 0:
-        return (
-            gr.update(interactive=False), 
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-        )
-
-    try:
-        ds = datasets.load_dataset(dataset_id, dataset_config, trust_remote_code=True)
-        df: pd.DataFrame = ds[dataset_split].to_pandas().head(5)
-        ds_labels, ds_features, _ = get_labels_and_features_from_dataset(ds[dataset_split])
-
-        if model_task is None or model_task != "text-classification":
-          gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
-          return (
-              gr.update(interactive=False), 
-              gr.update(value=df, visible=True),
-              gr.update(visible=False),
-              gr.update(visible=False),
-              gr.update(visible=False),
-              gr.update(visible=False),
-          )
-
-        if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
-            gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
-            return (
-                gr.update(interactive=False), 
-                gr.update(value=df, visible=True),
-                gr.update(visible=False),
-                gr.update(visible=False),
-                gr.update(visible=False),
-                gr.update(visible=False),
-            )
-
-        return (
-                gr.update(interactive=True), 
-                gr.update(value=df, visible=True),
-                gr.update(visible=False),
-                gr.update(visible=False),
-                gr.update(visible=False),
-                gr.update(visible=False),
-            )
-    except Exception as e:
-        # Config or split wrong
-        logger.warn(f"Check your dataset {dataset_id} and config {dataset_config} on split {dataset_split}: {e}")
-        return (
-            gr.update(interactive=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-        )
-
-
-def align_columns_and_show_prediction(
-    model_id,
-    dataset_id,
-    dataset_config,
-    dataset_split,
-    uid,
-    profile: gr.OAuthProfile | None,
-    oauth_token: gr.OAuthToken | None,
-):
-    model_id = strip_model_id_from_url(model_id)
-    model_task = check_model_task(model_id)
-    if model_task is None or model_task != "text-classification":
-        gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
-        return (
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False, open=False),
-            gr.update(interactive=False),
-            "",
-            *[gr.update(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)],
-        )
-
-    dropdown_placement = [
-        gr.Dropdown(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)
-    ]
-    
-    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
-
-    prediction_input, prediction_response = get_example_prediction(
-        model_id, dataset_id, dataset_config, dataset_split, hf_token
-    )
-
-    if prediction_input is None or prediction_response is None:
-        return (
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False, open=False),
-            gr.update(interactive=False),
-            "",
-            *dropdown_placement,
-        )
-
-    if isinstance(prediction_response, HuggingFaceInferenceAPIResponse):
-        return (
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False, open=False),
-            gr.update(interactive=False),
-            f"Hugging Face Inference API is loading your model. {prediction_response.message}",
-            *dropdown_placement,
-        )
-
-    model_labels = list(prediction_response.keys())
-    
-    ds = datasets.load_dataset(dataset_id, dataset_config, split=dataset_split, trust_remote_code=True)
-    ds_labels, ds_features, _ = get_labels_and_features_from_dataset(ds)
-
-    # when dataset does not have labels or features
-    if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
-        gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
-        return (
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False, open=False),
-            gr.update(interactive=False),
-            "",
-            *dropdown_placement,
-        )
-    
-    if len(ds_labels) != len(model_labels):
-        return (
-            gr.update(value=UNMATCHED_MODEL_DATASET_STYLED_ERROR, visible=True),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False, open=False),
-            gr.update(interactive=False),
-            "",
-            *dropdown_placement,
-        )
-
-    column_mappings = list_labels_and_features_from_dataset(
-        ds_labels,
-        ds_features,
-        model_labels,
-        uid,
-    )
-
-    # when labels or features are not aligned
-    # show manually column mapping
-    if (
-        collections.Counter(model_labels) != collections.Counter(ds_labels)
-        or ds_features[0] != "text"
-    ):
-        return (
-            gr.update(value=MAPPING_STYLED_ERROR_WARNING, visible=True),
-            gr.update(value=prediction_input, lines=min(len(prediction_input)//225 + 1, 5), visible=True),
-            gr.update(value=prediction_response, visible=True),
-            gr.update(visible=True, open=True),
-            gr.update(interactive=(profile is not None and oauth_token is not None)),
-            "",
-            *column_mappings,
-        )
-
-    return (
-        gr.update(value=VALIDATED_MODEL_DATASET_STYLED, visible=True),
-        gr.update(value=prediction_input, lines=min(len(prediction_input)//225 + 1, 5), visible=True),
-        gr.update(value=prediction_response, visible=True),
-        gr.update(visible=True, open=False),
-        gr.update(interactive=(profile is not None and oauth_token is not None)),
-        "",
-        *column_mappings,
-    )
-
-
-def check_column_mapping_keys_validity(all_mappings):
-    if all_mappings is None:
-        logger.warning("all_mapping is None")
-        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
-        return False
-
-    if "labels" not in all_mappings.keys():
-        logger.warning(f"Label mapping is not valid, all_mappings: {all_mappings}")
-        return False
-
-    return True
-
-def enable_run_btn(uid, model_id, dataset_id, dataset_config, dataset_split, profile: gr.OAuthProfile | None, oath_token: gr.OAuthToken | None):
-    if profile is None:
-        return gr.update(interactive=False)
-    if oath_token is None:
-        return gr.update(interactive=False)
-    if model_id == "" or dataset_id == "" or dataset_config == "" or dataset_split == "":
-        logger.warn("Model id or dataset id is not selected")
-        return gr.update(interactive=False)
-    
-    all_mappings = read_column_mapping(uid)
-    if not check_column_mapping_keys_validity(all_mappings):
-        logger.warn("Column mapping is not valid")
-        return gr.update(interactive=False)
-
-def construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features, label_keys=None):
-    label_mapping = {}
-    if len(all_mappings["labels"].keys()) != len(ds_labels):
-        logger.warn(f"""Label mapping corrupted: {CONFIRM_MAPPING_DETAILS_FAIL_RAW}. 
-                    \nall_mappings: {all_mappings}\nds_labels: {ds_labels}""")
-      
-    if len(all_mappings["features"].keys()) != len(ds_features):
-        logger.warn(f"""Feature mapping corrupted: {CONFIRM_MAPPING_DETAILS_FAIL_RAW}. 
-                    \nall_mappings: {all_mappings}\nds_features: {ds_features}""")
-
-    for i, label in zip(range(len(ds_labels)),  ds_labels):
-        # align the saved labels with dataset labels order
-        label_mapping.update({str(i): all_mappings["labels"][label]})
-
-    if "features" not in all_mappings.keys():
-        logger.warning("features not in all_mappings")
-        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
-
-    feature_mapping = all_mappings["features"]
-    if len(label_keys) > 0:
-      feature_mapping.update({"label": label_keys[0]})
-    return label_mapping, feature_mapping
-
-def show_hf_token_info(token):
-    valid = check_hf_token_validity(token)
-    if not valid:
-        return gr.update(visible=True)
-    return gr.update(visible=False)
-
-def try_submit(m_id, d_id, config, split, uid, profile: gr.OAuthProfile | None, oath_token: gr.OAuthToken | None):
-    all_mappings = read_column_mapping(uid)
-    if not check_column_mapping_keys_validity(all_mappings):
-        return (gr.update(interactive=True), gr.update(visible=False))
-
-    # get ds labels and features again for alignment
-    ds = datasets.load_dataset(d_id, config, split=split, trust_remote_code=True)
-    ds_labels, ds_features, label_keys = get_labels_and_features_from_dataset(ds)
-    label_mapping, feature_mapping = construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features, label_keys)
-    
-    eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
-    save_job_to_pipe(
-        uid,
-        (
-            m_id,
-            d_id,
-            config,
-            split,
-            oath_token.token,
-            uid,
-            label_mapping,
-            feature_mapping,
-        ),
-        eval_str,
-        threading.Lock(),
-    )
-    gr.Info("Your evaluation has been submitted")
-
-    new_uid = uuid.uuid4()
-    scanners = read_scanners(uid)
-    write_scanners(scanners, new_uid)
-
-    return (
-        gr.update(interactive=False),  # Submit button
-        gr.update(value=f"{CHECK_LOG_SECTION_RAW}Your job id is: {uid}. ", lines=5, visible=True, interactive=False),
-        new_uid,  # Allocate a new uuid
-        gr.update(visible=False),
-        gr.update(visible=False),
-        gr.update(visible=False),
-        gr.update(visible=False),
-    )

utils.py

@@ -1,29 +0,0 @@
-import sys
-
-import yaml
-
-
-# read scanners from yaml file
-# return a list of scanners
-def read_scanners(path):
-    scanners = []
-    with open(path, "r") as f:
-        config = yaml.load(f, Loader=yaml.FullLoader)
-        scanners = config.get("detectors", None)
-    return scanners
-
-
-# convert a list of scanners to yaml file
-def write_scanners(scanners):
-    with open("./scan_config.yaml", "w") as f:
-        # save scanners to detectors in yaml
-        yaml.dump({"detectors": scanners}, f)
-
-
-# convert column mapping dataframe to json
-def convert_column_mapping_to_json(df, label=""):
-    column_mapping = {}
-    column_mapping[label] = []
-    for _, row in df.iterrows():
-        column_mapping[label].append(row.tolist())
-    return column_mapping

utils/io_utils.py

@@ -1,25 +1,15 @@
 import os
-import logging
+
 import yaml
 
-YAML_PATH = "../cicd/configs"
-LOG_FILE = "../temp_log"
+YAML_PATH = "./cicd/configs"
+LOG_FILE = "temp_log"
 
-logger = logging.getLogger(__name__)
 
 class Dumper(yaml.Dumper):
     def increase_indent(self, flow=False, *args, **kwargs):
         return super().increase_indent(flow=flow, indentless=False)
 
-def get_submitted_yaml_path(uid):
-    if not os.path.exists(f"{YAML_PATH}/submitted"):
-        os.makedirs(f"{YAML_PATH}/submitted")
-    if not os.path.exists(f"{YAML_PATH}/{uid}_config.yaml"):
-        logger.error(f"config.yaml does not exist for {uid}")
-        os.system(f"cp config.yaml {YAML_PATH}/{uid}_config.yaml")
-    if not os.path.exists(f"{YAML_PATH}/submitted/{uid}_config.yaml"):
-        os.system(f"cp {YAML_PATH}/{uid}_config.yaml {YAML_PATH}/submitted/{uid}_config.yaml")
-    return f"{YAML_PATH}/submitted/{uid}_config.yaml"
 
 def get_yaml_path(uid):
     if not os.path.exists(YAML_PATH):
@@ -82,8 +72,6 @@ def read_column_mapping(uid):
         config = yaml.load(f, Loader=yaml.FullLoader)
         if config:
             column_mapping = config.get("column_mapping", dict())
-    if column_mapping is None:
-        column_mapping = {}
     return column_mapping
 
 

utils/run_jobs.py

@@ -17,7 +17,7 @@ from app_env import (
     HF_SPACE_ID,
     HF_WRITE_TOKEN,
 )
-from io_utils import LOG_FILE, get_submitted_yaml_path, write_log_to_user_file
+from utils.io_utils import LOG_FILE, get_yaml_path, write_log_to_user_file
 from isolated_env import prepare_venv
 from utils.leaderboard import LEADERBOARD
 
@@ -50,6 +50,7 @@ def prepare_env_and_get_command(
     d_id,
     config,
     split,
+    inference,
     inference_token,
     uid,
     label_mapping,
@@ -59,6 +60,10 @@ def prepare_env_and_get_command(
     if os.environ.get("SPACE_ID") == "giskardai/giskard-evaluator":
         leaderboard_dataset = LEADERBOARD
 
+    inference_type = "hf_pipeline"
+    if inference and inference_token:
+        inference_type = "hf_inference_api"
+
     executable = "giskard_scanner"
     try:
         # Copy the current requirements (might be changed)
@@ -93,9 +98,9 @@ def prepare_env_and_get_command(
         "--label_mapping",
         json.dumps(label_mapping),
         "--scan_config",
-        get_submitted_yaml_path(uid),
+        get_yaml_path(uid),
         "--inference_type",
-        "hf_inference_api",
+        inference_type,
         "--inference_api_token",
         inference_token,
     ]

utils/ui_helpers.py

@@ -7,15 +7,10 @@ import datasets
 import gradio as gr
 import pandas as pd
 
-import leaderboard
-from io_utils import (
-  read_column_mapping, 
-  write_column_mapping, 
-  read_scanners, 
-  write_scanners,
-)
-from run_jobs import save_job_to_pipe
-from text_classification import (
+import utils.leaderboard as leaderboard
+from utils.io_utils import read_column_mapping, write_column_mapping
+from utils.run_jobs import save_job_to_pipe
+from utils.text_classification import (
     strip_model_id_from_url,
     check_model_task,
     preload_hf_inference_api,
@@ -31,11 +26,10 @@ from utils.wordings import (
     NOT_TEXT_CLASSIFICATION_MODEL_RAW,
     UNMATCHED_MODEL_DATASET_STYLED_ERROR,
     CHECK_LOG_SECTION_RAW,
-    VALIDATED_MODEL_DATASET_STYLED,
+    get_styled_input,
     get_dataset_fetch_error_raw,
 )
 import os
-from app_env import HF_WRITE_TOKEN
 
 MAX_LABELS = 40
 MAX_FEATURES = 20
@@ -53,20 +47,9 @@ def get_related_datasets_from_leaderboard(model_id):
     datasets_unique = list(model_records["dataset_id"].unique())
 
     if len(datasets_unique) == 0:
-        return gr.update(choices=[])
+        return gr.update(choices=[], value="")
     
-    return gr.update(choices=datasets_unique)
-
-
-logger = logging.getLogger(__file__)
-
-def get_dataset_splits(dataset_id, dataset_config):
-    try:
-        splits = datasets.get_dataset_split_names(dataset_id, dataset_config, trust_remote_code=True)
-        return gr.update(choices=splits, value=splits[0], visible=True)
-    except Exception as e:
-        logger.warn(f"Check your dataset {dataset_id} and config {dataset_config}: {e}")
-        return gr.update(visible=False)
+    return gr.update(choices=datasets_unique, value="")
 
 def check_dataset(dataset_id):
     logger.info(f"Loading {dataset_id}")
@@ -78,7 +61,9 @@ def check_dataset(dataset_id):
                 gr.update(visible=False),
                 ""
             )
-        splits = datasets.get_dataset_split_names(dataset_id, configs[0], trust_remote_code=True)
+        splits = datasets.get_dataset_split_names(
+                        dataset_id, configs[0], trust_remote_code=True
+                    )
         return (
             gr.update(choices=configs, value=configs[0], visible=True),
             gr.update(choices=splits, value=splits[0], visible=True),
@@ -140,7 +125,7 @@ def list_labels_and_features_from_dataset(ds_labels, ds_features, model_labels,
         ds_labels = list(shared_labels)
     if len(ds_labels) > MAX_LABELS:
         ds_labels = ds_labels[:MAX_LABELS]
-        gr.Warning(f"Too many labels to display for this spcae. We do not support more than {MAX_LABELS} in this space. You can use cli tool at https://github.com/Giskard-AI/cicd.")
+        gr.Warning(f"The number of labels is truncated to length {MAX_LABELS}")
 
     # sort labels to make sure the order is consistent
     # prediction gives the order based on probability
@@ -181,67 +166,33 @@ def list_labels_and_features_from_dataset(ds_labels, ds_features, model_labels,
 
 def precheck_model_ds_enable_example_btn(
     model_id, dataset_id, dataset_config, dataset_split
-):  
+):
+    if model_id == "" or dataset_id == "":
+        return (gr.update(interactive=False), gr.update(visible=False), "")
     model_id = strip_model_id_from_url(model_id)
     model_task = check_model_task(model_id)
     preload_hf_inference_api(model_id)
-
+    if model_task is None or model_task != "text-classification":
+        gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
+        return (gr.update(interactive=False), gr.update(visible=False), "")
+    
     if dataset_config is None or dataset_split is None or len(dataset_config) == 0:
-        return (
-            gr.update(interactive=False), 
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-        )
-
+        return (gr.update(interactive=False), gr.update(visible=False), "")
+    
     try:
         ds = datasets.load_dataset(dataset_id, dataset_config, trust_remote_code=True)
         df: pd.DataFrame = ds[dataset_split].to_pandas().head(5)
-        ds_labels, ds_features, _ = get_labels_and_features_from_dataset(ds[dataset_split])
-
-        if model_task is None or model_task != "text-classification":
-          gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
-          return (
-              gr.update(interactive=False), 
-              gr.update(value=df, visible=True),
-              gr.update(visible=False),
-              gr.update(visible=False),
-              gr.update(visible=False),
-              gr.update(visible=False),
-          )
+        ds_labels, ds_features = get_labels_and_features_from_dataset(ds[dataset_split])
 
         if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
             gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
-            return (
-                gr.update(interactive=False), 
-                gr.update(value=df, visible=True),
-                gr.update(visible=False),
-                gr.update(visible=False),
-                gr.update(visible=False),
-                gr.update(visible=False),
-            )
+            return (gr.update(interactive=False), gr.update(value=df, visible=True), "")
 
-        return (
-                gr.update(interactive=True), 
-                gr.update(value=df, visible=True),
-                gr.update(visible=False),
-                gr.update(visible=False),
-                gr.update(visible=False),
-                gr.update(visible=False),
-            )
+        return (gr.update(interactive=True), gr.update(value=df, visible=True), "")
     except Exception as e:
         # Config or split wrong
         logger.warn(f"Check your dataset {dataset_id} and config {dataset_config} on split {dataset_split}: {e}")
-        return (
-            gr.update(interactive=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-        )
+        return (gr.update(interactive=False), gr.update(value=pd.DataFrame(), visible=False), "")
 
 
 def align_columns_and_show_prediction(
@@ -250,8 +201,8 @@ def align_columns_and_show_prediction(
     dataset_config,
     dataset_split,
     uid,
-    profile: gr.OAuthProfile | None,
-    oauth_token: gr.OAuthToken | None,
+    run_inference,
+    inference_token,
 ):
     model_id = strip_model_id_from_url(model_id)
     model_task = check_model_task(model_id)
@@ -270,7 +221,7 @@ def align_columns_and_show_prediction(
         gr.Dropdown(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)
     ]
     
-    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
+    hf_token = os.environ.get("HF_WRITE_TOKEN", default="")
 
     prediction_input, prediction_response = get_example_prediction(
         model_id, dataset_id, dataset_config, dataset_split, hf_token
@@ -278,7 +229,6 @@ def align_columns_and_show_prediction(
 
     if prediction_input is None or prediction_response is None:
         return (
-            gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False, open=False),
@@ -289,7 +239,6 @@ def align_columns_and_show_prediction(
 
     if isinstance(prediction_response, HuggingFaceInferenceAPIResponse):
         return (
-            gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False, open=False),
@@ -301,13 +250,12 @@ def align_columns_and_show_prediction(
     model_labels = list(prediction_response.keys())
     
     ds = datasets.load_dataset(dataset_id, dataset_config, split=dataset_split, trust_remote_code=True)
-    ds_labels, ds_features, _ = get_labels_and_features_from_dataset(ds)
+    ds_labels, ds_features = get_labels_and_features_from_dataset(ds)
 
     # when dataset does not have labels or features
     if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
         gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
         return (
-            gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False, open=False),
@@ -320,7 +268,6 @@ def align_columns_and_show_prediction(
         return (
             gr.update(value=UNMATCHED_MODEL_DATASET_STYLED_ERROR, visible=True),
             gr.update(visible=False),
-            gr.update(visible=False),
             gr.update(visible=False, open=False),
             gr.update(interactive=False),
             "",
@@ -342,20 +289,18 @@ def align_columns_and_show_prediction(
     ):
         return (
             gr.update(value=MAPPING_STYLED_ERROR_WARNING, visible=True),
-            gr.update(value=prediction_input, lines=min(len(prediction_input)//225 + 1, 5), visible=True),
-            gr.update(value=prediction_response, visible=True),
+            gr.update(visible=False),
             gr.update(visible=True, open=True),
-            gr.update(interactive=(profile is not None and oauth_token is not None)),
+            gr.update(interactive=(run_inference and inference_token != "")),
             "",
             *column_mappings,
         )
 
     return (
-        gr.update(value=VALIDATED_MODEL_DATASET_STYLED, visible=True),
-        gr.update(value=prediction_input, lines=min(len(prediction_input)//225 + 1, 5), visible=True),
+        gr.update(value=get_styled_input(prediction_input), visible=True),
         gr.update(value=prediction_response, visible=True),
         gr.update(visible=True, open=False),
-        gr.update(interactive=(profile is not None and oauth_token is not None)),
+        gr.update(interactive=(run_inference and inference_token != "")),
         "",
         *column_mappings,
     )
@@ -363,20 +308,18 @@ def align_columns_and_show_prediction(
 
 def check_column_mapping_keys_validity(all_mappings):
     if all_mappings is None:
-        logger.warning("all_mapping is None")
         gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
         return False
 
     if "labels" not in all_mappings.keys():
-        logger.warning(f"Label mapping is not valid, all_mappings: {all_mappings}")
+        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
         return False
 
     return True
 
-def enable_run_btn(uid, model_id, dataset_id, dataset_config, dataset_split, profile: gr.OAuthProfile | None, oath_token: gr.OAuthToken | None):
-    if profile is None:
-        return gr.update(interactive=False)
-    if oath_token is None:
+def enable_run_btn(uid, run_inference, inference_token, model_id, dataset_id, dataset_config, dataset_split):
+    if not run_inference or inference_token == "":
+        logger.warn("Inference API is not enabled")
         return gr.update(interactive=False)
     if model_id == "" or dataset_id == "" or dataset_config == "" or dataset_split == "":
         logger.warn("Model id or dataset id is not selected")
@@ -387,27 +330,26 @@ def enable_run_btn(uid, model_id, dataset_id, dataset_config, dataset_split, pro
         logger.warn("Column mapping is not valid")
         return gr.update(interactive=False)
 
-def construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features, label_keys=None):
+    if not check_hf_token_validity(inference_token):
+        logger.warn("HF token is not valid")
+        return gr.update(interactive=False)
+    return gr.update(interactive=True)
+
+def construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features):
     label_mapping = {}
     if len(all_mappings["labels"].keys()) != len(ds_labels):
-        logger.warn(f"""Label mapping corrupted: {CONFIRM_MAPPING_DETAILS_FAIL_RAW}. 
-                    \nall_mappings: {all_mappings}\nds_labels: {ds_labels}""")
+        logger.warn("Label mapping corrupted: " + CONFIRM_MAPPING_DETAILS_FAIL_RAW)
       
     if len(all_mappings["features"].keys()) != len(ds_features):
-        logger.warn(f"""Feature mapping corrupted: {CONFIRM_MAPPING_DETAILS_FAIL_RAW}. 
-                    \nall_mappings: {all_mappings}\nds_features: {ds_features}""")
+        logger.warn("Feature mapping corrupted: " + CONFIRM_MAPPING_DETAILS_FAIL_RAW)
 
     for i, label in zip(range(len(ds_labels)),  ds_labels):
         # align the saved labels with dataset labels order
         label_mapping.update({str(i): all_mappings["labels"][label]})
 
     if "features" not in all_mappings.keys():
-        logger.warning("features not in all_mappings")
         gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
-
     feature_mapping = all_mappings["features"]
-    if len(label_keys) > 0:
-      feature_mapping.update({"label": label_keys[0]})
     return label_mapping, feature_mapping
 
 def show_hf_token_info(token):
@@ -416,18 +358,16 @@ def show_hf_token_info(token):
         return gr.update(visible=True)
     return gr.update(visible=False)
 
-def try_submit(m_id, d_id, config, split, uid, profile: gr.OAuthProfile | None, oath_token: gr.OAuthToken | None):
-    print(oath_token.token)
-    print(".>>>>>>>>>>>>>>>>>>>>>>")
+def try_submit(m_id, d_id, config, split, inference, inference_token, uid):
     all_mappings = read_column_mapping(uid)
     if not check_column_mapping_keys_validity(all_mappings):
         return (gr.update(interactive=True), gr.update(visible=False))
 
     # get ds labels and features again for alignment
     ds = datasets.load_dataset(d_id, config, split=split, trust_remote_code=True)
-    ds_labels, ds_features, label_keys = get_labels_and_features_from_dataset(ds)
-    label_mapping, feature_mapping = construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features, label_keys)
-    
+    ds_labels, ds_features = get_labels_and_features_from_dataset(ds)
+    label_mapping, feature_mapping = construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features)
+
     eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
     save_job_to_pipe(
         uid,
@@ -436,7 +376,8 @@ def try_submit(m_id, d_id, config, split, uid, profile: gr.OAuthProfile | None,
             d_id,
             config,
             split,
-            oath_token.token,
+            inference,
+            inference_token,
             uid,
             label_mapping,
             feature_mapping,
@@ -446,16 +387,8 @@ def try_submit(m_id, d_id, config, split, uid, profile: gr.OAuthProfile | None,
     )
     gr.Info("Your evaluation has been submitted")
 
-    new_uid = uuid.uuid4()
-    scanners = read_scanners(uid)
-    write_scanners(scanners, new_uid)
-
     return (
         gr.update(interactive=False),  # Submit button
         gr.update(value=f"{CHECK_LOG_SECTION_RAW}Your job id is: {uid}. ", lines=5, visible=True, interactive=False),
-        new_uid,  # Allocate a new uuid
-        gr.update(visible=False),
-        gr.update(visible=False),
-        gr.update(visible=False),
-        gr.update(visible=False),
+        uuid.uuid4(),  # Allocate a new uuid
     )

utils/wordings.py

@@ -2,24 +2,23 @@ INTRODUCTION_MD = """
                 <h1 style="text-align: center;">
                 🐢Giskard Evaluator - Text Classification
                 </h1>
-                Welcome to the Giskard Evaluator Space! Get a model vulnerability report immediately by simply sharing your model and dataset id below. 
-                You can also checkout our library documentation <a href="https://docs.giskard.ai/en/latest/getting_started/quickstart/index.html">here</a>.
+                Welcome to the Giskard Evaluator Space! Get a model vulnerability report immediately by simply sharing your model and dataset id below.
                 """
 CONFIRM_MAPPING_DETAILS_MD = """
                             <h1 style="text-align: center;">
                             Confirm Pre-processing Details
                             </h1>
-                            Make sure the output variable's labels and the input variable's name are accurately mapped across both the dataset and the model. You can select the output variable's labels from the dropdowns below.
+                            Make sure the output variable's labels and the input variable's name are accurately mapped across both the dataset and the model.
                             """
 CONFIRM_MAPPING_DETAILS_FAIL_MD = """
                             <h1 style="text-align: center;">
                             Confirm Pre-processing Details
                             </h1>
-                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
+                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. <b>Please manually check the mapping below.</b>
                             """
 
 CONFIRM_MAPPING_DETAILS_FAIL_RAW = """
-                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
+                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. <b>Please manually check the mapping below.</b>
                             """
 
 CHECK_CONFIG_OR_SPLIT_RAW = """
@@ -39,7 +38,7 @@ PREDICTION_SAMPLE_MD = """
 
 MAPPING_STYLED_ERROR_WARNING = """
                         <h3 style="text-align: center;color: orange; background-color: #fff0f3; border-radius: 8px; padding: 10px; ">
-                        ⚠️ We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
+                        ⚠️ We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. <b>Please manually check the mapping below.</b>
                         </h3>
                         """
 
@@ -58,11 +57,7 @@ USE_INFERENCE_API_TIP = """
                 <a href="https://huggingface.co/docs/api-inference/detailed_parameters#text-classification-task">
                     Hugging Face Inference API
                 </a>
-                . Please input your <a href="https://huggingface.co/settings/tokens">Hugging Face token</a> to do so. You can find it <a href="https://huggingface.co/settings/tokens">here</a>.
-            """
-
-LOG_IN_TIPS = """
-            To use the Hugging Face Inference API, you need to log in to your Hugging Face account.
+                . Please input your <a href="https://huggingface.co/settings/tokens">Hugging Face token</a> to do so.
             """
 
 HF_TOKEN_INVALID_STYLED= """
@@ -71,10 +66,10 @@ HF_TOKEN_INVALID_STYLED= """
                 </p>
                 """
 
-VALIDATED_MODEL_DATASET_STYLED = """
-            <h3 style="text-align: center;color: #4ca154; background-color: #e2fbe8; border-radius: 8px; padding: 10px; ">
-            Your model and dataset have been validated!
-            </h3>"""
-
 def get_dataset_fetch_error_raw(error):
     return f"""Sorry you cannot use this dataset because {error}. Contact HF team to support this dataset."""
+
+def get_styled_input(input):
+    return f"""<h3 style="text-align: center;color: #4ca154; background-color: #e2fbe8; border-radius: 8px; padding: 10px; ">
+            Your model and dataset have been validated! <br /> Sample input: {input}
+            </h3>"""

wordings.py

@@ -1,80 +0,0 @@
-INTRODUCTION_MD = """
-                <div style="display: flex; justify-content: center;"><h1>
-                🐢Giskard Evaluator - Text Classification
-                </h1></div>
-                Welcome to the Giskard Evaluator Space! Get a model vulnerability report immediately by simply sharing your model and dataset id below. 
-                You can also checkout our library documentation <a href="https://docs.giskard.ai/en/latest/getting_started/quickstart/index.html">here</a>.
-                """
-CONFIRM_MAPPING_DETAILS_MD = """
-                            <h1 style="text-align: center;">
-                            Confirm Pre-processing Details
-                            </h1>
-                            Make sure the output variable's labels and the input variable's name are accurately mapped across both the dataset and the model. You can select the output variable's labels from the dropdowns below.
-                            """
-CONFIRM_MAPPING_DETAILS_FAIL_MD = """
-                            <h1 style="text-align: center;">
-                            Confirm Pre-processing Details
-                            </h1>
-                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
-                            """
-
-CONFIRM_MAPPING_DETAILS_FAIL_RAW = """
-                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
-                            """
-
-CHECK_CONFIG_OR_SPLIT_RAW = """
-                            Please check your dataset config or split.
-                            """
-
-CHECK_LOG_SECTION_RAW = """
-                          Your have successfully submitted a Giskard evaluation. Further details are available in the Logs tab. You can find your report will be posted to your model's community discussion.
-                        """
-
-PREDICTION_SAMPLE_MD = """
-                        <h1 style="text-align: center;">
-                        Model Prediction Sample
-                        </h1>
-                        Here's a sample of your model's prediction on an example from the dataset.
-                        """
-
-MAPPING_STYLED_ERROR_WARNING = """
-                        <h3 style="text-align: center;color: orange; background-color: #fff0f3; border-radius: 8px; padding: 10px; ">
-                        ⚠️ We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
-                        </h3>
-                        """
-
-UNMATCHED_MODEL_DATASET_STYLED_ERROR = """
-                        <h3 style="text-align: center;color: #fa5f5f; background-color: #fbe2e2; border-radius: 8px; padding: 10px; ">
-                        ❌ Your model and dataset have different numbers of labels. Please double check your model and dataset.
-                        </h3>
-                        """
-
-NOT_TEXT_CLASSIFICATION_MODEL_RAW = """
-                        Your model does not fall under the category of text classification. This page is specifically designated for the evaluation of text classification models.  
-                      """
-
-USE_INFERENCE_API_TIP = """
-                To speed up the evaluation, we recommend using the
-                <a href="https://huggingface.co/docs/api-inference/detailed_parameters#text-classification-task">
-                    Hugging Face Inference API
-                </a>
-                . Please input your <a href="https://huggingface.co/settings/tokens">Hugging Face token</a> to do so. You can find it <a href="https://huggingface.co/settings/tokens">here</a>.
-            """
-
-LOG_IN_TIPS = """
-            To use the Hugging Face Inference API, you need to log in to your Hugging Face account.
-            """
-
-HF_TOKEN_INVALID_STYLED= """
-                <p style="text-align: left;color: red; ">
-                Your Hugging Face token is invalid. Please double check your token.
-                </p>
-                """
-
-VALIDATED_MODEL_DATASET_STYLED = """
-            <h3 style="text-align: center;color: #4ca154; background-color: #e2fbe8; border-radius: 8px; padding: 10px; ">
-            Your model and dataset have been validated!
-            </h3>"""
-
-def get_dataset_fetch_error_raw(error):
-    return f"""Sorry you cannot use this dataset because {error}. Contact HF team to support this dataset."""