Update app.py

app.py

@@ -208,13 +208,13 @@ def visualize_logprobs(json_input, chunk=0, chunk_size=100):
                     logger.debug("top_logprobs is None for token: %s, using empty dict", token)
                     top_logprobs = {}  # Default to empty dict for None
                 # Ensure all values in top_logprobs are floats
-                finite_top_logprobs = []
+                finite_top_probs = []
                 for key, value in top_logprobs.items():
                     float_value = ensure_float(value)
                     if float_value is not None and math.isfinite(float_value):
-                        finite_top_logprobs.append((key, float_value))
+                        finite_top_probs.append((key, float_value))
                 # Sort by log probability (descending)
-                sorted_probs = sorted(finite_top_logprobs, key=lambda x: x[1], reverse=True)
+                sorted_probs = sorted(finite_top_probs, key=lambda x: x[1], reverse=True)
                 row = [token, f"{logprob:.4f}"]
                 for alt_token, alt_logprob in sorted_probs[:max_alternatives]:  # Use max number of alternatives
                     row.append(f"{alt_token}: {alt_logprob:.4f}")
@@ -281,93 +281,306 @@ def visualize_logprobs(json_input, chunk=0, chunk_size=100):
         logger.error("Visualization failed: %s (Input: %s)", str(e), json_input[:100] + "..." if len(json_input) > 100 else json_input)
         return (create_empty_figure("Log Probabilities of Generated Tokens"), None, "No finite log probabilities to display.", create_empty_figure("Top Token Log Probabilities"), create_empty_figure("Significant Probability Drops"), 1, 0)
 
-# Gradio interface with chunked visualization and proactive precomputation
+# Analysis functions for detecting correct vs. incorrect traces
+def analyze_confidence_signature(logprobs, tokens):
+    if not logprobs or not tokens:
+        return "No data for confidence signature analysis.", None
+    # Track moving average of top token probability
+    top_probs = [lps[0][1] if lps else -float('inf') for lps in logprobs]  # Extract top probability, handle empty
+    moving_avg = np.convolve(
+        top_probs,
+        np.ones(20) / 20,  # 20-token window
+        mode='valid'
+    )
+    
+    # Detect significant drops (potential error points)
+    drops = np.where(np.diff(moving_avg) < -0.15)[0]
+    if not drops.size:
+        return "No significant confidence drops detected.", None
+    drop_positions = [(i, tokens[i + 19] if i + 19 < len(tokens) else "End of trace") for i in drops]  # Adjust for convolution window
+    return "Significant confidence drops detected at positions:", drop_positions
+
+def detect_interpretation_pivots(logprobs, tokens):
+    if not logprobs or not tokens:
+        return "No data for interpretation pivot detection.", None
+    pivots = []
+    reconsideration_tokens = ["wait", "but", "actually", "however", "hmm"]
+    
+    for i, (token, lps) in enumerate(zip(tokens, logprobs)):
+        # Check if reconsideration tokens have unusually high probability
+        for rt in reconsideration_tokens:
+            for t, p in lps:
+                if t.lower() == rt and p > -2.5:  # High probability
+                    # Look back to find what's being reconsidered
+                    context = tokens[max(0, i-50):i]
+                    pivots.append((i, rt, context))
+    
+    if not pivots:
+        return "No interpretation pivots detected.", None
+    return "Interpretation pivots detected:", pivots
+
+def calculate_decision_entropy(logprobs):
+    if not logprobs:
+        return "No data for entropy spike detection.", None
+    # Calculate entropy at each token position
+    entropies = []
+    for lps in logprobs:
+        if not lps:
+            entropies.append(0.0)
+            continue
+        # Calculate entropy: -sum(p * log(p)) for each probability
+        probs = [math.exp(p) for _, p in lps]  # Convert log probs to probabilities
+        if not probs or sum(probs) == 0:
+            entropies.append(0.0)
+            continue
+        entropy = -sum(p * math.log(p) for p in probs if p > 0)
+        entropies.append(entropy)
+    
+    # Detect significant entropy spikes
+    baseline = np.percentile(entropies, 75) if entropies else 0.0
+    spikes = [i for i, e in enumerate(entropies) if e > baseline * 1.5 if baseline > 0]
+    
+    if not spikes:
+        return "No entropy spikes detected at decision points.", None
+    return "Entropy spikes detected at positions:", spikes
+
+def analyze_conclusion_competition(logprobs, tokens):
+    if not logprobs or not tokens:
+        return "No data for conclusion competition analysis.", None
+    # Find tokens related to conclusion
+    conclusion_indices = [i for i, t in enumerate(tokens) 
+                        if any(marker in t.lower() for marker in 
+                              ["therefore", "thus", "boxed", "answer"])]
+    
+    if not conclusion_indices:
+        return "No conclusion markers found in trace.", None
+    
+    # Analyze probability gap between top and second choices near conclusion
+    gaps = []
+    conclusion_idx = conclusion_indices[-1]
+    end_range = min(conclusion_idx + 50, len(logprobs))
+    for idx in range(conclusion_idx, end_range):
+        if idx < len(logprobs) and len(logprobs[idx]) >= 2:
+            top_prob = logprobs[idx][0][1] if logprobs[idx] else -float('inf')
+            second_prob = logprobs[idx][1][1] if len(logprobs[idx]) > 1 else -float('inf')
+            gap = top_prob - second_prob if top_prob != -float('inf') and second_prob != -float('inf') else 0.0
+            gaps.append(gap)
+    
+    if not gaps:
+        return "No conclusion competition data available.", None
+    mean_gap = np.mean(gaps)
+    return f"Mean probability gap at conclusion: {mean_gap:.4f} (higher indicates more confident conclusion)", None
+
+def analyze_verification_signals(logprobs, tokens):
+    if not logprobs or not tokens:
+        return "No data for verification signal analysis.", None
+    verification_terms = ["verify", "check", "confirm", "ensure", "double"]
+    verification_probs = []
+    
+    for lps in logprobs:
+        # Look for verification terms in top-k tokens
+        max_v_prob = -float('inf')
+        for token, prob in lps:
+            if any(v_term in token.lower() for v_term in verification_terms):
+                max_v_prob = max(max_v_prob, prob)
+        
+        if max_v_prob > -float('inf'):
+            verification_probs.append(max_v_prob)
+    
+    if not verification_probs:
+        return "No verification signals detected.", None
+    count, mean_prob = len(verification_probs), np.mean(verification_probs)
+    return f"Verification signals found: {count} instances, mean probability: {mean_prob:.4f}", None
+
+def detect_semantic_inversions(logprobs, tokens):
+    if not logprobs or not tokens:
+        return "No data for semantic inversion detection.", None
+    inversion_pairs = [
+        ("more", "less"), ("larger", "smaller"), 
+        ("winning", "losing"), ("increase", "decrease"),
+        ("greater", "lesser"), ("positive", "negative")
+    ]
+    
+    inversions = []
+    for i, (token, lps) in enumerate(zip(tokens, logprobs)):
+        for pos, neg in inversion_pairs:
+            if token.lower() == pos:
+                # Check if negative term has high probability
+                for t, p in lps:
+                    if t.lower() == neg and p > -3.0:  # High competitor
+                        inversions.append((i, pos, neg, p))
+            elif token.lower() == neg:
+                # Check if positive term has high probability
+                for t, p in lps:
+                    if t.lower() == pos and p > -3.0:  # High competitor
+                        inversions.append((i, neg, pos, p))
+    
+    if not inversions:
+        return "No semantic inversions detected.", None
+    return "Semantic inversions detected:", inversions
+
+# Function to perform full trace analysis
+def analyze_full_trace(json_input):
+    try:
+        data = parse_input(json_input)
+        content = data.get("content", []) if isinstance(data, dict) else data
+        if not isinstance(content, list):
+            raise ValueError("Content must be a list of entries")
+
+        tokens = []
+        logprobs = []
+        for entry in content:
+            if not isinstance(entry, dict):
+                logger.warning("Skipping non-dictionary entry: %s", entry)
+                continue
+            logprob = ensure_float(entry.get("logprob", None))
+            if logprob >= -100000:  # Include all entries with default 0.0
+                tokens.append(get_token(entry))
+                top_probs = entry.get("top_logprobs", {})
+                if top_probs is None:
+                    top_probs = {}
+                finite_top_probs = []
+                for key, value in top_probs.items():
+                    float_value = ensure_float(value)
+                    if float_value is not None and math.isfinite(float_value):
+                        finite_top_probs.append((key, float_value))
+                logprobs.append(finite_top_probs)
+
+        if not logprobs or not tokens:
+            return "No valid data for trace analysis.", None, None, None, None, None
+
+        # Perform all analyses
+        confidence_result, confidence_data = analyze_confidence_signature(logprobs, tokens)
+        pivot_result, pivot_data = detect_interpretation_pivots(logprobs, tokens)
+        entropy_result, entropy_data = calculate_decision_entropy(logprobs)
+        conclusion_result, conclusion_data = analyze_conclusion_competition(logprobs, tokens)
+        verification_result, verification_data = analyze_verification_signals(logprobs, tokens)
+        inversion_result, inversion_data = detect_semantic_inversions(logprobs, tokens)
+
+        # Format results for display
+        analysis_html = f"""
+        <h3>Trace Analysis Results</h3>
+        <ul>
+            <li><strong>Confidence Signature:</strong> {confidence_result}</li>
+            {f"<ul><li>Positions: {', '.join(str(pos) for pos, tok in confidence_data)}</li></ul>" if confidence_data else ""}
+            <li><strong>Interpretation Pivots:</strong> {pivot_result}</li>
+            {f"<ul><li>Positions: {', '.join(str(pos) for pos, _, _ in pivot_data)}</li></ul>" if pivot_data else ""}
+            <li><strong>Decision Entropy Spikes:</strong> {entropy_result}</li>
+            {f"<ul><li>Positions: {', '.join(str(pos) for pos in entropy_data)}</li></ul>" if entropy_data else ""}
+            <li><strong>Conclusion Competition:</strong> {conclusion_result}</li>
+            <li><strong>Verification Signals:</strong> {verification_result}</li>
+            <li><strong>Semantic Inversions:</strong> {inversion_result}</li>
+            {f"<ul><li>Positions: {', '.join(str(pos) for pos, _, _, _ in inversion_data)}</li></ul>" if inversion_data else ""}
+        </ul>
+        """
+        return analysis_html, None, None, None, None, None
+
+# Gradio interface with two tabs: Trace Analysis and Visualization
 with gr.Blocks(title="Log Probability Visualizer") as app:
     gr.Markdown("# Log Probability Visualizer")
     gr.Markdown(
-        "Paste your JSON log prob data below to visualize tokens in chunks of 100. Fixed filter ≥ -100000, dynamic number of top_logprobs, handles missing or null fields. Next chunk is precomputed proactively."
+        "Paste your JSON log prob data below to analyze reasoning traces and visualize tokens in chunks of 100. Fixed filter ≥ -100000, dynamic number of top_logprobs, handles missing or null fields. Next chunk is precomputed proactively."
     )
 
-    with gr.Row():
-        json_input = gr.Textbox(
-            label="JSON Input",
-            lines=10,
-            placeholder="Paste your JSON (e.g., {\"content\": [{\"bytes\": [44], \"logprob\": 0.0, \"token\": \",\", \"top_logprobs\": {\" so\": -13.8046875, \".\": -13.8046875, \"，\": -13.640625}}]}).",
-        )
-        chunk = gr.Number(value=0, label="Current Chunk", precision=0, minimum=0)
-
-    with gr.Row():
-        plot_output = gr.Plot(label="Log Probability Plot (Click for Tokens)")
-        drops_output = gr.Plot(label="Probability Drops (Click for Details)")
-
-    with gr.Row():
-        table_output = gr.Dataframe(label="Token Log Probabilities and Top Alternatives")
-        alt_viz_output = gr.Plot(label="Top Token Log Probabilities (Click for Details)")
-
-    with gr.Row():
-        text_output = gr.HTML(label="Colored Text (Confidence Visualization)")
-
-    with gr.Row():
-        prev_btn = gr.Button("Previous Chunk")
-        next_btn = gr.Button("Next Chunk")
-        total_chunks_output = gr.Number(label="Total Chunks", interactive=False)
-
-    # Precomputed next chunk state (hidden)
-    precomputed_next = gr.State(value=None)
+    with gr.Tabs():
+        with gr.Tab("Trace Analysis"):
+            with gr.Row():
+                json_input_analysis = gr.Textbox(
+                    label="JSON Input for Trace Analysis",
+                    lines=10,
+                    placeholder="Paste your JSON (e.g., {\"content\": [{\"bytes\": [44], \"logprob\": 0.0, \"token\": \",\", \"top_logprobs\": {\" so\": -13.8046875, \".\": -13.8046875, \"，\": -13.640625}}]}).",
+                )
+            with gr.Row():
+                analysis_output = gr.HTML(label="Trace Analysis Results")
+
+            btn_analyze = gr.Button("Analyze Trace")
+            btn_analyze.click(
+                fn=analyze_full_trace,
+                inputs=[json_input_analysis],
+                outputs=[analysis_output, gr.State(), gr.State(), gr.State(), gr.State(), gr.State()],
+            )
 
-    btn = gr.Button("Visualize")
-    btn.click(
-        fn=visualize_logprobs,
-        inputs=[json_input, chunk],
-        outputs=[plot_output, table_output, text_output, alt_viz_output, drops_output, total_chunks_output, chunk],
-    )
+        with gr.Tab("Visualization"):
+            with gr.Row():
+                json_input_viz = gr.Textbox(
+                    label="JSON Input for Visualization",
+                    lines=10,
+                    placeholder="Paste your JSON (e.g., {\"content\": [{\"bytes\": [44], \"logprob\": 0.0, \"token\": \",\", \"top_logprobs\": {\" so\": -13.8046875, \".\": -13.8046875, \"，\": -13.640625}}]}).",
+                )
+                chunk = gr.Number(value=0, label="Current Chunk", precision=0, minimum=0)
+
+            with gr.Row():
+                plot_output = gr.Plot(label="Log Probability Plot (Click for Tokens)")
+                drops_output = gr.Plot(label="Probability Drops (Click for Details)")
+
+            with gr.Row():
+                table_output = gr.Dataframe(label="Token Log Probabilities and Top Alternatives")
+                alt_viz_output = gr.Plot(label="Top Token Log Probabilities (Click for Details)")
+
+            with gr.Row():
+                text_output = gr.HTML(label="Colored Text (Confidence Visualization)")
+
+            with gr.Row():
+                prev_btn = gr.Button("Previous Chunk")
+                next_btn = gr.Button("Next Chunk")
+                total_chunks_output = gr.Number(label="Total Chunks", interactive=False)
+
+            # Precomputed next chunk state (hidden)
+            precomputed_next = gr.State(value=None)
+
+            btn_viz = gr.Button("Visualize")
+            btn_viz.click(
+                fn=visualize_logprobs,
+                inputs=[json_input_viz, chunk],
+                outputs=[plot_output, table_output, text_output, alt_viz_output, drops_output, total_chunks_output, chunk],
+            )
 
-    # Precompute next chunk proactively when on current chunk
-    async def precompute_next_chunk(json_input, current_chunk, precomputed_next):
-        if precomputed_next is not None:
-            return precomputed_next  # Use cached precomputed chunk if available
-        next_tokens, next_logprobs, next_alternatives = await precompute_chunk(json_input, 100, current_chunk)
-        if next_tokens is None or next_logprobs is None or next_alternatives is None:
-            return None
-        return (next_tokens, next_logprobs, next_alternatives)
-
-    # Update chunk on button clicks
-    def update_chunk(json_input, current_chunk, action, precomputed_next=None):
-        total_chunks = visualize_logprobs(json_input, 0)[5]  # Get total chunks
-        if action == "prev" and current_chunk > 0:
-            current_chunk -= 1
-        elif action == "next" and current_chunk < total_chunks - 1:
-            current_chunk += 1
-            # If precomputed next chunk exists, use it; otherwise, compute it
-            if precomputed_next:
-                next_tokens, next_logprobs, next_alternatives = precomputed_next
-                if next_tokens and next_logprobs and next_alternatives:
-                    logger.debug("Using precomputed next chunk for chunk %d", current_chunk)
-                    return visualize_logprobs(json_input, current_chunk)
-        return visualize_logprobs(json_input, current_chunk)
-
-    prev_btn.click(
-        fn=update_chunk,
-        inputs=[json_input, chunk, gr.State(value="prev"), precomputed_next],
-        outputs=[plot_output, table_output, text_output, alt_viz_output, drops_output, total_chunks_output, chunk],
-    )
+            # Precompute next chunk proactively when on current chunk
+            async def precompute_next_chunk(json_input, current_chunk, precomputed_next):
+                if precomputed_next is not None:
+                    return precomputed_next  # Use cached precomputed chunk if available
+                next_tokens, next_logprobs, next_alternatives = await precompute_chunk(json_input, 100, current_chunk)
+                if next_tokens is None or next_logprobs is None or next_alternatives is None:
+                    return None
+                return (next_tokens, next_logprobs, next_alternatives)
+
+            # Update chunk on button clicks
+            def update_chunk(json_input, current_chunk, action, precomputed_next=None):
+                total_chunks = visualize_logprobs(json_input, 0)[5]  # Get total chunks
+                if action == "prev" and current_chunk > 0:
+                    current_chunk -= 1
+                elif action == "next" and current_chunk < total_chunks - 1:
+                    current_chunk += 1
+                    # If precomputed next chunk exists, use it; otherwise, compute it
+                    if precomputed_next:
+                        next_tokens, next_logprobs, next_alternatives = precomputed_next
+                        if next_tokens and next_logprobs and next_alternatives:
+                            logger.debug("Using precomputed next chunk for chunk %d", current_chunk)
+                            return visualize_logprobs(json_input, current_chunk)
+                return visualize_logprobs(json_input, current_chunk)
+
+            prev_btn.click(
+                fn=update_chunk,
+                inputs=[json_input_viz, chunk, gr.State(value="prev"), precomputed_next],
+                outputs=[plot_output, table_output, text_output, alt_viz_output, drops_output, total_chunks_output, chunk],
+            )
 
-    next_btn.click(
-        fn=update_chunk,
-        inputs=[json_input, chunk, gr.State(value="next"), precomputed_next],
-        outputs=[plot_output, table_output, text_output, alt_viz_output, drops_output, total_chunks_output, chunk],
-    )
+            next_btn.click(
+                fn=update_chunk,
+                inputs=[json_input_viz, chunk, gr.State(value="next"), precomputed_next],
+                outputs=[plot_output, table_output, text_output, alt_viz_output, drops_output, total_chunks_output, chunk],
+            )
 
-    # Trigger precomputation when chunk changes (via button clicks or initial load)
-    def trigger_precomputation(json_input, current_chunk):
-        asyncio.create_task(precompute_next_chunk(json_input, current_chunk, None))
-        return gr.update(value=current_chunk)
+            # Trigger precomputation when chunk changes (via button clicks or initial load)
+            def trigger_precomputation(json_input, current_chunk):
+                asyncio.create_task(precompute_next_chunk(json_input, current_chunk, None))
+                return gr.update(value=current_chunk)
 
-    # Use a dummy event to trigger precomputation on chunk change (simplified for Gradio)
-    chunk.change(
-        fn=trigger_precomputation,
-        inputs=[json_input, chunk],
-        outputs=[chunk],
-    )
+            # Use a dummy event to trigger precomputation on chunk change (simplified for Gradio)
+            chunk.change(
+                fn=trigger_precomputation,
+                inputs=[json_input_viz, chunk],
+                outputs=[chunk],
+            )
 
 app.launch()