import re import logging from dataclasses import dataclass from typing import Optional, Dict, Any from datetime import datetime, timedelta logger = logging.getLogger(__name__) @dataclass class TrainingState: """Represents the current state of training""" status: str = "idle" # idle, initializing, training, completed, error, stopped current_step: int = 0 total_steps: int = 0 current_epoch: int = 0 total_epochs: int = 0 step_loss: float = 0.0 learning_rate: float = 0.0 grad_norm: float = 0.0 memory_allocated: float = 0.0 memory_reserved: float = 0.0 start_time: Optional[datetime] = None last_step_time: Optional[datetime] = None estimated_remaining: Optional[timedelta] = None error_message: Optional[str] = None initialization_stage: str = "" download_progress: float = 0.0 def calculate_progress(self) -> float: """Calculate overall progress as percentage""" if self.total_steps == 0: return 0.0 return (self.current_step / self.total_steps) * 100 def to_dict(self) -> Dict[str, Any]: """Convert state to dictionary for UI updates""" # Calculate elapsed time only if training is active and we have a start time if self.start_time and self.status in ["training", "initializing"]: elapsed = str(datetime.now() - self.start_time) else: # Use the last known elapsed time or show 0 elapsed = "0:00:00" if not self.last_step_time else str(self.last_step_time - self.start_time if self.start_time else "0:00:00") # Use precomputed remaining time from logs if available remaining = str(self.estimated_remaining) if self.estimated_remaining else "calculating..." return { "status": self.status, "progress": f"{self.calculate_progress():.1f}%", "current_step": self.current_step, "total_steps": self.total_steps, "current_epoch": self.current_epoch, "total_epochs": self.total_epochs, "step_loss": f"{self.step_loss:.4f}", "learning_rate": f"{self.learning_rate:.2e}", "grad_norm": f"{self.grad_norm:.4f}", "memory": f"{self.memory_allocated:.1f}GB allocated, {self.memory_reserved:.1f}GB reserved", "elapsed": elapsed, "remaining": remaining, "initialization_stage": self.initialization_stage, "error_message": self.error_message, "download_progress": self.download_progress } class TrainingLogParser: """Parser for training logs with state management""" def __init__(self): self.state = TrainingState() self._last_update_time = None def parse_line(self, line: str) -> Optional[Dict[str, Any]]: """Parse a single log line and update state""" try: # For debugging #logger.info(f"Parsing line: {line[:100]}...") # Training step progress line example: # Training steps: 1%|▏ | 1/70 [00:14<16:11, 14.08s/it, grad_norm=0.00789, step_loss=0.555, lr=3e-7] if ("Started training" in line) or ("Starting training" in line): self.state.status = "training" # Check for "Training steps:" which contains the progress information if "Training steps:" in line: # Set status to training if we see this self.state.status = "training" if not self.state.start_time: self.state.start_time = datetime.now() # Extract step numbers steps_match = re.search(r"(\d+)/(\d+)", line) if steps_match: self.state.current_step = int(steps_match.group(1)) self.state.total_steps = int(steps_match.group(2)) # Extract metrics for pattern, attr in [ (r"step_loss=([0-9.e-]+)", "step_loss"), (r"lr=([0-9.e-]+)", "learning_rate"), (r"grad_norm=([0-9.e-]+)", "grad_norm") ]: match = re.search(pattern, line) if match: setattr(self.state, attr, float(match.group(1))) # Extract time remaining directly from the log # Format: [MM:SS