Don't commitschedule when locally + use local database for testing
Browse files
app.py
CHANGED
@@ -6,7 +6,7 @@ import gradio as gr
|
|
6 |
import numpy as np
|
7 |
from PIL import Image
|
8 |
import random
|
9 |
-
from db import compute_elo_scores, get_all_votes, add_vote
|
10 |
import json
|
11 |
from pathlib import Path
|
12 |
from uuid import uuid4
|
@@ -16,6 +16,8 @@ import time
|
|
16 |
from datasets import load_dataset
|
17 |
from huggingface_hub import CommitScheduler
|
18 |
|
|
|
|
|
19 |
token = os.getenv("HUGGINGFACE_HUB_TOKEN")
|
20 |
|
21 |
# Load datasets
|
@@ -31,15 +33,16 @@ load_dotenv()
|
|
31 |
JSON_DATASET_DIR = Path("data/json_dataset")
|
32 |
JSON_DATASET_DIR.mkdir(parents=True, exist_ok=True)
|
33 |
|
34 |
-
# Initialize CommitScheduler for Hugging Face
|
35 |
-
scheduler =
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
|
|
43 |
|
44 |
def fetch_elo_scores():
|
45 |
"""Fetch and log Elo scores."""
|
@@ -216,8 +219,10 @@ def gradio_interface():
|
|
216 |
try:
|
217 |
logging.debug("Adding vote data to the database: %s", vote_data)
|
218 |
|
219 |
-
|
220 |
-
|
|
|
|
|
221 |
except Exception as e:
|
222 |
logging.error("Error recording vote: %s", str(e))
|
223 |
|
@@ -270,6 +275,10 @@ def gradio_interface():
|
|
270 |
|
271 |
def dump_database_to_json():
|
272 |
"""Dump the database to a JSON file and upload it to Hugging Face."""
|
|
|
|
|
|
|
|
|
273 |
votes = get_all_votes()
|
274 |
json_data = [
|
275 |
{
|
@@ -301,10 +310,13 @@ def schedule_dump_database(interval=60):
|
|
301 |
logging.info("Database dump completed. Sleeping for %d seconds.", interval)
|
302 |
time.sleep(interval)
|
303 |
|
304 |
-
|
305 |
-
|
306 |
-
|
307 |
-
|
|
|
|
|
|
|
308 |
|
309 |
if __name__ == "__main__":
|
310 |
schedule_dump_database() # Start the periodic database dump
|
|
|
6 |
import numpy as np
|
7 |
from PIL import Image
|
8 |
import random
|
9 |
+
from db import compute_elo_scores, get_all_votes, add_vote, is_running_in_space
|
10 |
import json
|
11 |
from pathlib import Path
|
12 |
from uuid import uuid4
|
|
|
16 |
from datasets import load_dataset
|
17 |
from huggingface_hub import CommitScheduler
|
18 |
|
19 |
+
|
20 |
+
|
21 |
token = os.getenv("HUGGINGFACE_HUB_TOKEN")
|
22 |
|
23 |
# Load datasets
|
|
|
33 |
JSON_DATASET_DIR = Path("data/json_dataset")
|
34 |
JSON_DATASET_DIR.mkdir(parents=True, exist_ok=True)
|
35 |
|
36 |
+
# Initialize CommitScheduler for Hugging Face only if running in space
|
37 |
+
scheduler = None
|
38 |
+
if is_running_in_space():
|
39 |
+
scheduler = CommitScheduler(
|
40 |
+
repo_id="bgsys/votes_datasets_test2",
|
41 |
+
repo_type="dataset",
|
42 |
+
folder_path=JSON_DATASET_DIR,
|
43 |
+
path_in_repo="data",
|
44 |
+
token=token
|
45 |
+
)
|
46 |
|
47 |
def fetch_elo_scores():
|
48 |
"""Fetch and log Elo scores."""
|
|
|
219 |
try:
|
220 |
logging.debug("Adding vote data to the database: %s", vote_data)
|
221 |
|
222 |
+
# Only add vote if running in space
|
223 |
+
if is_running_in_space():
|
224 |
+
result = add_vote(vote_data)
|
225 |
+
logging.info("Vote successfully recorded in the database with ID: %s", result["id"])
|
226 |
except Exception as e:
|
227 |
logging.error("Error recording vote: %s", str(e))
|
228 |
|
|
|
275 |
|
276 |
def dump_database_to_json():
|
277 |
"""Dump the database to a JSON file and upload it to Hugging Face."""
|
278 |
+
if not is_running_in_space():
|
279 |
+
logging.info("Not running in Hugging Face Spaces. Skipping database dump.")
|
280 |
+
return
|
281 |
+
|
282 |
votes = get_all_votes()
|
283 |
json_data = [
|
284 |
{
|
|
|
310 |
logging.info("Database dump completed. Sleeping for %d seconds.", interval)
|
311 |
time.sleep(interval)
|
312 |
|
313 |
+
if is_running_in_space():
|
314 |
+
logging.info("Initializing database dump scheduler with interval: %d seconds.", interval)
|
315 |
+
thread = threading.Thread(target=run, daemon=True)
|
316 |
+
thread.start()
|
317 |
+
logging.info("Database dump scheduler started.")
|
318 |
+
else:
|
319 |
+
logging.info("Not running in Hugging Face Spaces. Database dump scheduler not started.")
|
320 |
|
321 |
if __name__ == "__main__":
|
322 |
schedule_dump_database() # Start the periodic database dump
|
db.py
CHANGED
@@ -1,3 +1,4 @@
|
|
|
|
1 |
from sqlalchemy import create_engine, Column, Integer, String, DateTime
|
2 |
from sqlalchemy.ext.declarative import declarative_base
|
3 |
from sqlalchemy.orm import sessionmaker, Session
|
@@ -6,8 +7,14 @@ import pandas as pd
|
|
6 |
import uuid
|
7 |
from rating_systems import compute_elo
|
8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
9 |
|
10 |
-
DATABASE_URL = "sqlite:///./data/newvotes.db" # Example with SQLite, replace with PostgreSQL for production
|
11 |
engine = create_engine(DATABASE_URL, connect_args={"check_same_thread": False})
|
12 |
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
13 |
Base = declarative_base()
|
|
|
1 |
+
import os
|
2 |
from sqlalchemy import create_engine, Column, Integer, String, DateTime
|
3 |
from sqlalchemy.ext.declarative import declarative_base
|
4 |
from sqlalchemy.orm import sessionmaker, Session
|
|
|
7 |
import uuid
|
8 |
from rating_systems import compute_elo
|
9 |
|
10 |
+
def is_running_in_space():
|
11 |
+
return "SPACE_ID" in os.environ
|
12 |
+
|
13 |
+
if is_running_in_space():
|
14 |
+
DATABASE_URL = "sqlite:///./data/newvotes.db"
|
15 |
+
else:
|
16 |
+
DATABASE_URL = "sqlite:///./data/local.db"
|
17 |
|
|
|
18 |
engine = create_engine(DATABASE_URL, connect_args={"check_same_thread": False})
|
19 |
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
20 |
Base = declarative_base()
|