File size: 2,790 Bytes
e49f146 7247691 e49f146 977ad93 e49f146 bb28e75 e49f146 9b6304b e49f146 9b6304b bb28e75 9b6304b 977ad93 bb28e75 e49f146 bb28e75 e49f146 bb28e75 9b6304b e49f146 bb28e75 9b6304b e49f146 bb28e75 9b6304b 7247691 c434d74 7247691 c434d74 7247691 977ad93 7247691 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
import logging
import os
import shutil
import datetime
import threading
import time
logger = logging.getLogger(__name__)
def purge_old_runs(run_dir: str, max_age_hours: float = 1.0, prefix: str = "myrun_") -> None:
"""
Deletes files and directories in the specified run_dir older than max_age_hours and matching the specified prefix.
"""
if not os.path.isabs(run_dir):
raise ValueError(f"run_dir must be an absolute path: {run_dir}")
if not os.path.exists(run_dir):
logger.error(f"run_dir does not exist: {run_dir} -- skipping purge")
return
logger.info("Running purge...")
now = datetime.datetime.now()
cutoff = now - datetime.timedelta(hours=max_age_hours)
count_deleted = 0
count_skip_without_prefix = 0
count_skip_recent = 0
count_error = 0
for item in os.listdir(run_dir):
if not item.startswith(prefix):
count_skip_without_prefix += 1
continue # Skip files and directories that don't match the prefix
item_path = os.path.join(run_dir, item)
try:
# Get the modification time of the item (file or directory)
mtime = datetime.datetime.fromtimestamp(os.path.getmtime(item_path))
if mtime < cutoff:
logger.debug(f"Deleting old data: {item} from {run_dir}")
if os.path.isdir(item_path):
shutil.rmtree(item_path) # Delete the directory and all its contents
else:
os.remove(item_path) # Delete the file
count_deleted += 1
else:
logger.debug(f"Skipping {item} in {run_dir}, last modified: {mtime}")
count_skip_recent += 1
except Exception as e:
logger.error(f"Error processing {item} in {run_dir}: {e}")
count_error += 1
logger.info(f"Purge complete: {count_deleted} deleted, {count_skip_recent} skipped (recent), {count_skip_without_prefix} skipped (no prefix), {count_error} errors")
def start_purge_scheduler(run_dir: str, purge_interval_seconds: float = 3600, prefix: str = "myrun_") -> None:
"""
Start the purge scheduler in a background thread.
"""
logger.info(f"Starting purge scheduler for {run_dir} every {purge_interval_seconds} seconds. Prefix: {prefix}")
if not os.path.isabs(run_dir):
raise ValueError(f"run_dir must be an absolute path: {run_dir}")
def purge_scheduler():
"""
Schedules the purge_old_runs function to run periodically.
"""
while True:
purge_old_runs(run_dir, prefix=prefix)
time.sleep(purge_interval_seconds)
purge_thread = threading.Thread(target=purge_scheduler, daemon=True)
purge_thread.start()
|