no

app.py

@@ -313,7 +313,7 @@ def prepare_dpo_dataset(df):
     return pd.DataFrame(dpo_data)
 
 
-def train_model(train_df, val_df, epochs=3, batch_size=4, lr=5e-5, max_samples=None):
+def train_model(train_df, val_df, epochs=3, batch_size=4, lr=2e-5, max_samples=None):
     """Training with DPO (Direct Preference Optimization)"""
     global current_model, current_tokenizer
     
@@ -415,7 +415,7 @@ def train_model(train_df, val_df, epochs=3, batch_size=4, lr=5e-5, max_samples=N
         report_to=[],
         max_length=seq_length,
         max_prompt_length=seq_length,
-        beta=0.1,
+        beta=1.0,  # Increased from 0.1 for stronger preference learning
         optim="adamw_8bit" if current_model_id == "openai/gpt-oss-20b" else "adamw_torch",
         dataloader_num_workers=2,  # A100 can handle parallel loading
     )
@@ -443,8 +443,65 @@ def train_model(train_df, val_df, epochs=3, batch_size=4, lr=5e-5, max_samples=N
     
     # Custom callback for status updates
     from transformers import TrainerCallback
+    import numpy as np
+    
+    def compute_accuracy_metrics(trainer, eval_dataset, num_samples=100):
+        """Compute accuracy metrics on a subset of eval data"""
+        # Sample subset for faster evaluation
+        indices = np.random.choice(len(eval_dataset), min(num_samples, len(eval_dataset)), replace=False)
+        
+        predictions_yes = 0
+        predictions_no = 0
+        correct = 0
+        
+        for idx in indices:
+            item = eval_dataset[int(idx)]
+            prompt = item['prompt']
+            true_choice = item['chosen']  # This is the correct answer
+            
+            # Tokenize and run inference
+            inputs = current_tokenizer(prompt, return_tensors="pt", truncation=True, max_length=512)
+            inputs = {k: v.to(trainer.model.device) for k, v in inputs.items()}
+            
+            with torch.no_grad():
+                outputs = trainer.model(**inputs)
+                logits = outputs.logits[0, -1, :]
+                
+                # Get token IDs
+                yes_token_id = current_tokenizer.encode("yes", add_special_tokens=False)[0]
+                no_token_id = current_tokenizer.encode("no", add_special_tokens=False)[0]
+                
+                yes_logit = logits[yes_token_id].item()
+                no_logit = logits[no_token_id].item()
+                
+                # Get prediction
+                prediction = " yes" if yes_logit > no_logit else " no"
+                
+                if prediction == " yes":
+                    predictions_yes += 1
+                else:
+                    predictions_no += 1
+                
+                if prediction.strip() == true_choice.strip():
+                    correct += 1
+        
+        accuracy = correct / len(indices)
+        yes_ratio = predictions_yes / len(indices)
+        no_ratio = predictions_no / len(indices)
+        
+        return {
+            'accuracy': accuracy,
+            'yes_ratio': yes_ratio,
+            'no_ratio': no_ratio,
+            'total_samples': len(indices)
+        }
     
     class StatusCallback(TrainerCallback):
+        def __init__(self, trainer, eval_dataset):
+            self.trainer = trainer
+            self.eval_dataset = eval_dataset
+            self.eval_every_n_steps = 50  # Evaluate every 50 steps
+            
         def on_log(self, args, state, control, logs=None, **kwargs):
             if logs:
                 with training_lock:
@@ -452,13 +509,32 @@ def train_model(train_df, val_df, epochs=3, batch_size=4, lr=5e-5, max_samples=N
                         training_status["logs"].append(f"Step {state.global_step}: Loss = {logs['loss']:.4f}")
                     if "eval_loss" in logs:
                         training_status["logs"].append(f"Eval Loss = {logs['eval_loss']:.4f}")
+                    
+                    # Compute accuracy metrics periodically
+                    if state.global_step > 0 and state.global_step % self.eval_every_n_steps == 0:
+                        metrics = compute_accuracy_metrics(self.trainer, self.eval_dataset)
+                        training_status["logs"].append(
+                            f"Step {state.global_step} Metrics: "
+                            f"Accuracy={metrics['accuracy']:.2%}, "
+                            f"Yes={metrics['yes_ratio']:.1%}, "
+                            f"No={metrics['no_ratio']:.1%}"
+                        )
+                        
+                        # Warn if model is biased
+                        if metrics['yes_ratio'] < 0.2 or metrics['no_ratio'] < 0.2:
+                            training_status["logs"].append(
+                                f"⚠️ WARNING: Model is heavily biased! "
+                                f"(Yes: {metrics['yes_ratio']:.1%}, No: {metrics['no_ratio']:.1%})"
+                            )
+                    
                     # Update progress
                     if state.global_step > 0:
                         total_steps = len(train_dataset) // batch_size * epochs
                         training_status["progress"] = min(int((state.global_step / total_steps) * 100), 99)
     
-    # Add callback
-    dpo_trainer.add_callback(StatusCallback())
+    # Add callback with trainer and eval dataset
+    status_callback = StatusCallback(dpo_trainer, val_dataset)
+    dpo_trainer.add_callback(status_callback)
     
     # Train
     try:
@@ -471,6 +547,18 @@ def train_model(train_df, val_df, epochs=3, batch_size=4, lr=5e-5, max_samples=N
         current_tokenizer.save_pretrained(save_path)
         logger.info(f"Model saved to {save_path}")
         
+        # Compute final metrics
+        logger.info("Computing final accuracy metrics...")
+        final_metrics = compute_accuracy_metrics(dpo_trainer, val_dataset, num_samples=200)
+        logger.info(f"Final Accuracy: {final_metrics['accuracy']:.2%}")
+        logger.info(f"Final Prediction Distribution - Yes: {final_metrics['yes_ratio']:.1%}, No: {final_metrics['no_ratio']:.1%}")
+        
+        with training_lock:
+            training_status["logs"].append(f"\n=== FINAL RESULTS ===")
+            training_status["logs"].append(f"Accuracy: {final_metrics['accuracy']:.2%}")
+            training_status["logs"].append(f"Yes predictions: {final_metrics['yes_ratio']:.1%}")
+            training_status["logs"].append(f"No predictions: {final_metrics['no_ratio']:.1%}")
+        
         # Update global model reference
         current_model = dpo_trainer.model
         current_model.eval()
@@ -546,7 +634,7 @@ def run_training(csv_path, shuffle_flag=False, split_ratio=0.8):
             max_samples = 2000  # Start conservative
         else:
             max_samples = None
-        train_model(train_df, test_df, epochs=3, batch_size=32, lr=5e-5, max_samples=max_samples)
+        train_model(train_df, test_df, epochs=3, batch_size=32, lr=2e-5, max_samples=max_samples)
         
         with training_lock:
             training_status["status"] = "completed"