Update app.py

app.py

@@ -308,25 +308,93 @@ def detect_interpretation_pivots(logprobs, tokens):
         return "No interpretation pivots detected.", None
     return "Interpretation pivots detected:", pivots
 
-def calculate_decision_entropy(logprobs):
+def calculate_decision_entropy(logprobs, tokens=None):
     if not logprobs:
         return "No data for entropy spike detection.", None
+    
+    # Calculate entropy at each position
     entropies = []
     for lps in logprobs:
-        if not lps:
+        if not lps or len(lps) < 2:  # Need at least two tokens for meaningful entropy
             entropies.append(0.0)
             continue
-        probs = [math.exp(p) for _, p in lps if p is not None]  # Convert log probs to probabilities, handle None
+            
+        # Only use top-5 tokens for entropy calculation to reduce noise
+        top_k = min(5, len(lps))
+        probs = [math.exp(p) for _, p in lps[:top_k] if p is not None]
+        
+        # Normalize probabilities to sum to 1
         if not probs or sum(probs) == 0:
             entropies.append(0.0)
             continue
-        entropy = -sum(p * math.log(p) for p in probs if p > 0)
+            
+        prob_sum = sum(probs)
+        normalized_probs = [p/prob_sum for p in probs]
+        
+        entropy = -sum(p * math.log(p) for p in normalized_probs if p > 0)
         entropies.append(entropy)
-    baseline = np.percentile(entropies, 75) if entropies else 0.0
-    spikes = [i for i, e in enumerate(entropies) if e > baseline * 1.5 and baseline > 0]
+    
+    # Smooth entropy values with moving average
+    window_size = 15
+    if len(entropies) >= window_size:
+        smoothed_entropies = np.convolve(entropies, np.ones(window_size)/window_size, mode='valid')
+    else:
+        smoothed_entropies = entropies
+    
+    # More selective threshold - 90th percentile and 2x multiplier
+    baseline = np.percentile(smoothed_entropies, 90) if smoothed_entropies.size > 0 else 0.0
+    
+    # Find significant spikes (much more selective)
+    spikes = []
+    if baseline > 0:
+        raw_spikes = np.where(smoothed_entropies > baseline * 2.0)[0]
+        
+        # Cluster nearby spikes (within 20 tokens)
+        if raw_spikes.size > 0:
+            spikes = [raw_spikes[0]]
+            for spike in raw_spikes[1:]:
+                if spike - spikes[-1] > 20:
+                    spikes.append(spike)
+    
+    # If we have token information, check context around spikes
+    if tokens and spikes:
+        context_spikes = []
+        decision_markers = ["therefore", "thus", "so", "hence", "because", 
+                          "wait", "but", "however", "actually", "instead"]
+        
+        for spike in spikes:
+            # Adjust index for convolution window if using smoothed values
+            spike_idx = spike + window_size//2 if len(entropies) >= window_size else spike
+            
+            if spike_idx >= len(tokens):
+                continue
+                
+            # Check surrounding context (15 tokens before and after)
+            start_idx = max(0, spike_idx - 15)
+            end_idx = min(len(tokens), spike_idx + 15)
+            
+            if end_idx <= start_idx:
+                continue
+                
+            context = " ".join(tokens[start_idx:end_idx])
+            
+            # Only keep spikes near reasoning transitions
+            if any(marker in context.lower() for marker in decision_markers):
+                entropy_value = smoothed_entropies[spike - window_size//2] if len(entropies) >= window_size else entropies[spike]
+                context_spikes.append((spike_idx, entropy_value, tokens[spike_idx] if spike_idx < len(tokens) else "End"))
+        
+        spikes = context_spikes
+    
+    # Return at most 3 most significant spikes
     if not spikes:
-        return "No entropy spikes detected at decision points.", None
-    return "Entropy spikes detected at positions:", spikes
+        return "No significant entropy spikes detected at decision points.", None
+    
+    # Sort by entropy value (highest first) if we have context information
+    if tokens and spikes:
+        spikes.sort(key=lambda x: x[1], reverse=True)
+        return "Significant entropy spikes detected at positions:", spikes[:3]
+    
+    return "Entropy spikes detected at positions:", spikes[:3]
 
 def analyze_conclusion_competition(logprobs, tokens):
     if not logprobs or not tokens: