Simon Strandgaard
commited on
Commit
·
9b6304b
1
Parent(s):
c434d74
Print stats after a purge
Browse files- src/purge/purge_old_runs.py +14 -2
src/purge/purge_old_runs.py
CHANGED
@@ -14,11 +14,21 @@ def purge_old_runs(run_dir: str, max_age_hours: float = 1.0, prefix: str = "myru
|
|
14 |
if not os.path.isabs(run_dir):
|
15 |
raise ValueError(f"run_dir must be an absolute path: {run_dir}")
|
16 |
|
|
|
|
|
|
|
|
|
|
|
17 |
now = datetime.datetime.now()
|
18 |
cutoff = now - datetime.timedelta(hours=max_age_hours)
|
19 |
|
|
|
|
|
|
|
|
|
20 |
for item in os.listdir(run_dir):
|
21 |
if not item.startswith(prefix):
|
|
|
22 |
continue # Skip files and directories that don't match the prefix
|
23 |
|
24 |
item_path = os.path.join(run_dir, item)
|
@@ -33,11 +43,15 @@ def purge_old_runs(run_dir: str, max_age_hours: float = 1.0, prefix: str = "myru
|
|
33 |
shutil.rmtree(item_path) # Delete the directory and all its contents
|
34 |
else:
|
35 |
os.remove(item_path) # Delete the file
|
|
|
36 |
else:
|
37 |
logger.debug(f"Skipping {item} in {run_dir}, last modified: {mtime}")
|
|
|
38 |
|
39 |
except Exception as e:
|
40 |
logger.error(f"Error processing {item} in {run_dir}: {e}")
|
|
|
|
|
41 |
|
42 |
def start_purge_scheduler(run_dir: str, purge_interval_seconds: float = 3600, prefix: str = "myrun_") -> None:
|
43 |
"""
|
@@ -53,8 +67,6 @@ def start_purge_scheduler(run_dir: str, purge_interval_seconds: float = 3600, pr
|
|
53 |
Schedules the purge_old_runs function to run periodically.
|
54 |
"""
|
55 |
while True:
|
56 |
-
logger.info("Running purge...")
|
57 |
-
print("Running purge...")
|
58 |
purge_old_runs(run_dir, prefix=prefix)
|
59 |
time.sleep(purge_interval_seconds)
|
60 |
|
|
|
14 |
if not os.path.isabs(run_dir):
|
15 |
raise ValueError(f"run_dir must be an absolute path: {run_dir}")
|
16 |
|
17 |
+
if not os.path.exists(run_dir):
|
18 |
+
logger.error(f"run_dir does not exist: {run_dir} -- skipping purge")
|
19 |
+
return
|
20 |
+
|
21 |
+
logger.info("Running purge...")
|
22 |
now = datetime.datetime.now()
|
23 |
cutoff = now - datetime.timedelta(hours=max_age_hours)
|
24 |
|
25 |
+
count_deleted = 0
|
26 |
+
count_skip_without_prefix = 0
|
27 |
+
count_skip_recent = 0
|
28 |
+
count_error = 0
|
29 |
for item in os.listdir(run_dir):
|
30 |
if not item.startswith(prefix):
|
31 |
+
count_skip_without_prefix += 1
|
32 |
continue # Skip files and directories that don't match the prefix
|
33 |
|
34 |
item_path = os.path.join(run_dir, item)
|
|
|
43 |
shutil.rmtree(item_path) # Delete the directory and all its contents
|
44 |
else:
|
45 |
os.remove(item_path) # Delete the file
|
46 |
+
count_deleted += 1
|
47 |
else:
|
48 |
logger.debug(f"Skipping {item} in {run_dir}, last modified: {mtime}")
|
49 |
+
count_skip_recent += 1
|
50 |
|
51 |
except Exception as e:
|
52 |
logger.error(f"Error processing {item} in {run_dir}: {e}")
|
53 |
+
count_error += 1
|
54 |
+
logger.info(f"Purge complete: {count_deleted} deleted, {count_skip_recent} skipped (recent), {count_skip_without_prefix} skipped (no prefix), {count_error} errors")
|
55 |
|
56 |
def start_purge_scheduler(run_dir: str, purge_interval_seconds: float = 3600, prefix: str = "myrun_") -> None:
|
57 |
"""
|
|
|
67 |
Schedules the purge_old_runs function to run periodically.
|
68 |
"""
|
69 |
while True:
|
|
|
|
|
70 |
purge_old_runs(run_dir, prefix=prefix)
|
71 |
time.sleep(purge_interval_seconds)
|
72 |
|