Spaces:
Running
Running
Commit
·
c198bfa
1
Parent(s):
50c2dfa
feat: dynamic local detecting within same repo
Browse files- app.py +2 -29
- src/envs.py +16 -7
- src/submission/submit.py +2 -2
app.py
CHANGED
@@ -3,7 +3,6 @@ from gradio.components import Dataframe
|
|
3 |
from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
|
4 |
import pandas as pd
|
5 |
from apscheduler.schedulers.background import BackgroundScheduler
|
6 |
-
from huggingface_hub import snapshot_download
|
7 |
import os
|
8 |
from gradio.themes import Soft
|
9 |
|
@@ -23,10 +22,9 @@ from src.display.utils import (
|
|
23 |
EVAL_COLS,
|
24 |
EVAL_TYPES,
|
25 |
AutoEvalColumn,
|
26 |
-
auto_eval_column_attrs
|
27 |
-
Language,
|
28 |
)
|
29 |
-
from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH,
|
30 |
from src.populate import get_evaluation_queue_df, get_leaderboard_df
|
31 |
from src.submission.submit import add_new_eval
|
32 |
|
@@ -50,31 +48,6 @@ def initialize_data_directories():
|
|
50 |
os.makedirs(EVAL_REQUESTS_PATH, exist_ok=True)
|
51 |
os.makedirs(EVAL_RESULTS_PATH, exist_ok=True)
|
52 |
|
53 |
-
if LOCAL_MODE:
|
54 |
-
print("Running in local mode, using local directories only")
|
55 |
-
return
|
56 |
-
|
57 |
-
# Try to download from HF if not in local mode
|
58 |
-
try:
|
59 |
-
print(f"Downloading request data from {QUEUE_REPO} to {EVAL_REQUESTS_PATH}")
|
60 |
-
snapshot_download(
|
61 |
-
repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset",
|
62 |
-
tqdm_class=None, etag_timeout=30, token=TOKEN
|
63 |
-
)
|
64 |
-
except Exception as e:
|
65 |
-
print(f"Failed to download request data: {e}")
|
66 |
-
print("Using local data only")
|
67 |
-
|
68 |
-
try:
|
69 |
-
print(f"Downloading result data from {RESULTS_REPO} to {EVAL_RESULTS_PATH}")
|
70 |
-
snapshot_download(
|
71 |
-
repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset",
|
72 |
-
tqdm_class=None, etag_timeout=30, token=TOKEN
|
73 |
-
)
|
74 |
-
except Exception as e:
|
75 |
-
print(f"Failed to download result data: {e}")
|
76 |
-
print("Using local data only")
|
77 |
-
|
78 |
# Initialize data
|
79 |
initialize_data_directories()
|
80 |
|
|
|
3 |
from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
|
4 |
import pandas as pd
|
5 |
from apscheduler.schedulers.background import BackgroundScheduler
|
|
|
6 |
import os
|
7 |
from gradio.themes import Soft
|
8 |
|
|
|
22 |
EVAL_COLS,
|
23 |
EVAL_TYPES,
|
24 |
AutoEvalColumn,
|
25 |
+
auto_eval_column_attrs
|
|
|
26 |
)
|
27 |
+
from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, REPO_ID, LOCAL_MODE
|
28 |
from src.populate import get_evaluation_queue_df, get_leaderboard_df
|
29 |
from src.submission.submit import add_new_eval
|
30 |
|
|
|
48 |
os.makedirs(EVAL_REQUESTS_PATH, exist_ok=True)
|
49 |
os.makedirs(EVAL_RESULTS_PATH, exist_ok=True)
|
50 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
51 |
# Initialize data
|
52 |
initialize_data_directories()
|
53 |
|
src/envs.py
CHANGED
@@ -2,21 +2,30 @@ import os
|
|
2 |
|
3 |
from huggingface_hub import HfApi
|
4 |
|
5 |
-
#
|
6 |
-
|
7 |
-
|
|
|
|
|
|
|
|
|
8 |
|
9 |
# Info to change for your repository
|
10 |
# ----------------------------------
|
11 |
# Get token from environment or use None in local mode
|
12 |
TOKEN = os.environ.get("HF_TOKEN") if not LOCAL_MODE else None
|
13 |
|
14 |
-
OWNER = "
|
15 |
# ----------------------------------
|
16 |
|
17 |
-
REPO_ID = f"{OWNER}/
|
18 |
-
QUEUE_REPO =
|
19 |
-
RESULTS_REPO =
|
|
|
|
|
|
|
|
|
|
|
20 |
|
21 |
# If you setup a cache later, just change HF_HOME
|
22 |
CACHE_PATH=os.getenv("HF_HOME", ".")
|
|
|
2 |
|
3 |
from huggingface_hub import HfApi
|
4 |
|
5 |
+
# Dynamically determine if we're running in local mode
|
6 |
+
def is_local_mode():
|
7 |
+
if os.environ.get("SPACE_AUTHOR_NAME") and os.environ.get("SPACE_REPO_NAME") and os.environ.get("HF_TOKEN") and os.environ.get("SPACE_ID"):
|
8 |
+
return False
|
9 |
+
return True
|
10 |
+
|
11 |
+
LOCAL_MODE = is_local_mode()
|
12 |
|
13 |
# Info to change for your repository
|
14 |
# ----------------------------------
|
15 |
# Get token from environment or use None in local mode
|
16 |
TOKEN = os.environ.get("HF_TOKEN") if not LOCAL_MODE else None
|
17 |
|
18 |
+
OWNER = "holistic-ai" # Change to your org - don't forget to create a results and request dataset, with the correct format!
|
19 |
# ----------------------------------
|
20 |
|
21 |
+
REPO_ID = f"{OWNER}/LibVulnWatch"
|
22 |
+
QUEUE_REPO = REPO_ID # Use the same repository
|
23 |
+
RESULTS_REPO = REPO_ID # Use the same repository
|
24 |
+
|
25 |
+
if not LOCAL_MODE:
|
26 |
+
REPO_ID = str(os.environ.get("SPACE_ID"))
|
27 |
+
QUEUE_REPO = REPO_ID
|
28 |
+
RESULTS_REPO = REPO_ID
|
29 |
|
30 |
# If you setup a cache later, just change HF_HOME
|
31 |
CACHE_PATH=os.getenv("HF_HOME", ".")
|
src/submission/submit.py
CHANGED
@@ -65,9 +65,9 @@ def add_new_eval(
|
|
65 |
path = Path(request_file_path)
|
66 |
API.upload_file(
|
67 |
path_or_fileobj=path,
|
68 |
-
path_in_repo=request_filename,
|
69 |
repo_id=QUEUE_REPO,
|
70 |
-
repo_type="
|
71 |
)
|
72 |
|
73 |
return styled_message(f"Library '{library_name}' has been added to the assessment queue! Assessment ID: {uid}")
|
|
|
65 |
path = Path(request_file_path)
|
66 |
API.upload_file(
|
67 |
path_or_fileobj=path,
|
68 |
+
path_in_repo=f"assessment-queue/{request_filename}",
|
69 |
repo_id=QUEUE_REPO,
|
70 |
+
repo_type="space",
|
71 |
)
|
72 |
|
73 |
return styled_message(f"Library '{library_name}' has been added to the assessment queue! Assessment ID: {uid}")
|