Create analyzer.py

analyzer.py

@@ -1,9 +1,9 @@
 import pandas as pd
 import numpy as np
 import logging
-from datetime import datetime
 import traceback
 from collections import Counter
+from typing import Dict, List, Tuple, Any
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -31,8 +31,29 @@ class MessageAnalyzer:
             "false equivalence": 0.317,
             "future faking": 0.385
         }
+        
+        # Fallacy severity weights for enhanced scoring
+        self.fallacy_weights = {
+            'Ad Hominem': 1.2,
+            'Strawman': 1.0,
+            'Whataboutism': 0.8,
+            'Gaslighting': 1.8,
+            'False Dichotomy': 1.0,
+            'Appeal to Emotion': 1.3,
+            'DARVO': 1.9,
+            'Moving Goalposts': 1.1,
+            'Cherry Picking': 0.9,
+            'Appeal to Authority': 0.7,
+            'Slippery Slope': 0.8,
+            'Motte and Bailey': 1.0,
+            'Gish Gallop': 1.1,
+            'Kafkatrapping': 1.4,
+            'Sealioning': 1.0,
+            'No Fallacy': 0.0
+        }
+
     def analyze_message(self, text):
-        """Analyze a single message for abuse patterns"""
+        """Enhanced message analysis incorporating fallacy detection"""
         from utils import (
             detect_explicit_abuse, detect_enhanced_threats, get_emotional_tone_tag,
             compute_abuse_score, get_boundary_assessment, calculate_enhanced_risk_level
@@ -42,368 +63,409 @@ class MessageAnalyzer:
         
         try:
             if not text.strip():
-                logger.debug("Empty text, returning zeros")
-                return {
-                    'abuse_score': 0.0,
-                    'detected_patterns': [],
-                    'matched_scores': [],
-                    'sentiment': "neutral",
-                    'sentiment_confidence': 0.5,
-                    'stage': 1,
-                    'darvo_score': 0.0,
-                    'emotional_tone': "neutral",
-                    'boundary_assessment': {'assessment': 'neutral', 'confidence': 0.5},
-                    'risk_level': "Low"
-                }
-            
-            # Check for explicit abuse
-            explicit_abuse = detect_explicit_abuse(text)
-            logger.debug(f"Explicit abuse detected: {explicit_abuse}")
-            
-            # Get sentiment
-            sentiment, sentiment_confidence = self.model_manager.predict_sentiment(text)
-            logger.debug(f"Sentiment: {sentiment} (confidence: {sentiment_confidence:.3f})")
-            
-            # Get boundary health
-            boundary_health = self.model_manager.predict_boundary_health(text)
-            boundary_assessment = get_boundary_assessment(text, boundary_health)
-            logger.debug(f"Boundary health: {boundary_assessment['assessment']}")
-            
-            # Early supportive message check
-            innocent_indicators = [
-                'broken', 'not working', 'cracked', 'glass', 'screen', 'phone',
-                'device', 'battery', 'charger', 'wifi', 'internet', 'computer',
-                'sorry', 'apologize', 'my fault', 'mistake'
-            ]
-                
-            # Enhanced early return check - now includes boundary health
-            if (any(indicator in text.lower() for indicator in innocent_indicators) and 
-                len(text.split()) < 20 and 
-                not any(threat in text.lower() for threat in ['kill', 'hurt', 'destroy', 'hate']) and
-                boundary_health > 0):  # Healthy boundary
-            
-                # If sentiment is strongly supportive AND boundary health is good, return early
-                if sentiment == "supportive" and sentiment_confidence > 0.8:
-                    logger.debug("Early return: Message appears to be innocent/supportive with healthy boundaries")
-                    return {
-                        'abuse_score': 0.0,
-                        'detected_patterns': [],
-                        'matched_scores': [],
-                        'sentiment': sentiment,
-                        'sentiment_confidence': sentiment_confidence,
-                        'stage': 1,
-                        'darvo_score': 0.0,
-                        'emotional_tone': "neutral",
-                        'boundary_assessment': boundary_assessment,
-                        'risk_level': "Low"
-                    }
-            
-            # Get abuse patterns
-            threshold_labels, matched_scores = self.model_manager.predict_abuse_patterns(text, self.thresholds)
-            logger.debug(f"Detected patterns: {threshold_labels}")
+                return self._get_empty_analysis()
             
-            # Check for enhanced threats
-            enhanced_patterns = detect_enhanced_threats(text, threshold_labels)
-            for pattern in enhanced_patterns:
-                if pattern not in threshold_labels:
-                    threshold_labels.append(pattern)
-                    # Add to matched_scores with high confidence
-                    weight = self.model_manager.get_pattern_weight(pattern)
-                    matched_scores.append((pattern, 0.85, weight))
+            # Get base analysis using your existing models
+            base_analysis = self._get_base_analysis(text)
             
-            # Get DARVO score
-            darvo_score = self.model_manager.predict_darvo(text)
-            logger.debug(f"DARVO score: {darvo_score:.3f}")
+            # Add fallacy analysis using FallacyFinder
+            fallacy_type, fallacy_confidence = self.model_manager.predict_fallacy(text)
             
-            # Get emotions
-            emotions = self.model_manager.get_emotion_profile(text)
-            logger.debug(f"Emotions: {emotions}")
-            
-            # Calculate abuse score
-            abuse_score = compute_abuse_score(matched_scores, sentiment)
-            logger.debug(f"Abuse score: {abuse_score:.1f}")
-            
-            # Apply explicit abuse override
-            if explicit_abuse:
-                abuse_score = max(abuse_score, 70.0)
-                if "insults" not in threshold_labels:
-                    threshold_labels.append("insults")
-                    matched_scores.append(("insults", 0.9, 1.4))
-            
-            # Apply boundary health modifier to abuse score
-            if boundary_health > 0 and not explicit_abuse:
-                # Healthy boundaries - cap abuse score lower
-                abuse_score = min(abuse_score, 35.0)
-                logger.debug(f"Capped abuse score to {abuse_score} due to healthy boundaries")
-            
-            # Apply sentiment-based score capping
-            if sentiment == "supportive" and not explicit_abuse:
-                # For supportive messages, cap the abuse score much lower
-                abuse_score = min(abuse_score, 30.0)
-                logger.debug(f"Capped abuse score to {abuse_score} due to supportive sentiment")
-            
-            # Get emotional tone
-            emotional_tone = get_emotional_tone_tag(text, sentiment, threshold_labels, abuse_score, emotions)
-            logger.debug(f"Emotional tone: {emotional_tone}")
+            # Enhanced intent analysis incorporating fallacy context
+            enhanced_intent = self._analyze_enhanced_intent(
+                text, fallacy_type, base_analysis['detected_patterns']
+            )
             
-            # Set stage
-            stage = 2 if explicit_abuse or abuse_score > 70 else 1
+            # Calculate enhanced abuse score with fallacy impact
+            enhanced_abuse_score = self._calculate_enhanced_abuse_score(
+                base_analysis['abuse_score'], 
+                fallacy_type, 
+                fallacy_confidence,
+                base_analysis['darvo_score']
+            )
             
-            # Calculate risk level
-            risk_level = calculate_enhanced_risk_level(
-                abuse_score, 
-                threshold_labels,
-                "Low" if abuse_score < 50 else "Moderate" if abuse_score < 70 else "High",
-                darvo_score
+            # Determine likely abuser indicators
+            abuser_indicators = self._calculate_abuser_indicators(
+                base_analysis, fallacy_type, fallacy_confidence
             )
             
+            # Enhanced analysis result
             return {
-                'abuse_score': abuse_score,
-                'detected_patterns': threshold_labels,
-                'matched_scores': matched_scores,
-                'sentiment': sentiment,
-                'sentiment_confidence': sentiment_confidence,
-                'stage': stage,
-                'darvo_score': darvo_score,
-                'emotional_tone': emotional_tone,
-                'boundary_assessment': boundary_assessment,
-                'risk_level': risk_level
+                **base_analysis,
+                'enhanced_abuse_score': enhanced_abuse_score,
+                'fallacy_detected': fallacy_type,
+                'fallacy_confidence': fallacy_confidence,
+                'enhanced_intent': enhanced_intent,
+                'abuser_indicators': abuser_indicators,
+                'manipulation_score': self._calculate_manipulation_score(
+                    base_analysis['detected_patterns'], fallacy_type
+                )
             }
             
         except Exception as e:
             logger.error(f"Error in analyze_message: {e}")
             logger.error(traceback.format_exc())
+            return self._get_error_analysis()
+
+    def _get_base_analysis(self, text):
+        """Get base analysis using existing trained models"""
+        from utils import (
+            detect_explicit_abuse, detect_enhanced_threats, get_emotional_tone_tag,
+            compute_abuse_score, get_boundary_assessment, calculate_enhanced_risk_level
+        )
+        
+        # Check for explicit abuse
+        explicit_abuse = detect_explicit_abuse(text)
+        
+        # Get predictions from your trained models
+        sentiment, sentiment_confidence = self.model_manager.predict_sentiment(text)
+        boundary_health = self.model_manager.predict_boundary_health(text)
+        boundary_assessment = get_boundary_assessment(text, boundary_health)
+        
+        # Early return for clearly supportive messages
+        if self._is_clearly_supportive(text, sentiment, sentiment_confidence, boundary_health):
             return {
                 'abuse_score': 0.0,
                 'detected_patterns': [],
                 'matched_scores': [],
-                'sentiment': "error",
-                'sentiment_confidence': 0.0,
+                'sentiment': sentiment,
+                'sentiment_confidence': sentiment_confidence,
                 'stage': 1,
                 'darvo_score': 0.0,
-                'emotional_tone': "error",
-                'boundary_assessment': {'assessment': 'error', 'confidence': 0.0},
-                'risk_level': "Unknown"
+                'emotional_tone': "neutral",
+                'boundary_assessment': boundary_assessment,
+                'risk_level': "Low"
             }
-    def identify_primary_abuser(self, results_df):
-        """Identify the primary abuser based on comprehensive abuse metrics with pattern severity weighting"""
-        logger.info("Identifying primary abuser...")
-        
-        # Define pattern severity weights (higher = more concerning)
-        PATTERN_WEIGHTS = {
-            "recovery phase": 0.7,
-            "control": 1.4,
-            "gaslighting": 1.3,
-            "guilt tripping": 1.2,
-            "dismissiveness": 0.9,
-            "blame shifting": 1.0,
-            "projection": 0.5,
-            "insults": 1.4,
-            "contradictory statements": 1.0,
-            "obscure language": 0.9,
-            "nonabusive": 0.0,  # Zero weight for nonabusive pattern
-            "veiled threats": 1.6,
-            "stalking language": 1.8,
-            "false concern": 1.1,
-            "false equivalence": 1.3,
-            "future faking": 0.8
+        
+        # Get abuse patterns from multilabel model
+        threshold_labels, matched_scores = self.model_manager.predict_abuse_patterns(text, self.thresholds)
+        
+        # Enhanced threat detection
+        enhanced_patterns = detect_enhanced_threats(text, threshold_labels)
+        for pattern in enhanced_patterns:
+            if pattern not in threshold_labels:
+                threshold_labels.append(pattern)
+                weight = self.model_manager.get_pattern_weight(pattern)
+                matched_scores.append((pattern, 0.85, weight))
+        
+        # Get DARVO score
+        darvo_score = self.model_manager.predict_darvo(text)
+        
+        # Get emotions
+        emotions = self.model_manager.get_emotion_profile(text)
+        
+        # Calculate abuse score
+        abuse_score = compute_abuse_score(matched_scores, sentiment)
+        
+        # Apply explicit abuse override
+        if explicit_abuse:
+            abuse_score = max(abuse_score, 70.0)
+            if "insults" not in threshold_labels:
+                threshold_labels.append("insults")
+                matched_scores.append(("insults", 0.9, 1.4))
+        
+        # Apply boundary and sentiment modifiers
+        abuse_score = self._apply_score_modifiers(
+            abuse_score, boundary_health, sentiment, explicit_abuse
+        )
+        
+        # Get emotional tone
+        emotional_tone = get_emotional_tone_tag(text, sentiment, threshold_labels, abuse_score, emotions)
+        
+        # Set stage and risk level
+        stage = 2 if explicit_abuse or abuse_score > 70 else 1
+        risk_level = calculate_enhanced_risk_level(
+            abuse_score, threshold_labels,
+            "Low" if abuse_score < 50 else "Moderate" if abuse_score < 70 else "High",
+            darvo_score
+        )
+        
+        return {
+            'abuse_score': abuse_score,
+            'detected_patterns': threshold_labels,
+            'matched_scores': matched_scores,
+            'sentiment': sentiment,
+            'sentiment_confidence': sentiment_confidence,
+            'stage': stage,
+            'darvo_score': darvo_score,
+            'emotional_tone': emotional_tone,
+            'boundary_assessment': boundary_assessment,
+            'risk_level': risk_level
         }
+
+    def _is_clearly_supportive(self, text, sentiment, sentiment_confidence, boundary_health):
+        """Check if message is clearly supportive and non-abusive"""
+        innocent_indicators = [
+            'broken', 'not working', 'cracked', 'glass', 'screen', 'phone',
+            'device', 'battery', 'charger', 'wifi', 'internet', 'computer',
+            'sorry', 'apologize', 'my fault', 'mistake'
+        ]
+        
+        threat_indicators = ['kill', 'hurt', 'destroy', 'hate']
+        
+        return (
+            any(indicator in text.lower() for indicator in innocent_indicators) and 
+            len(text.split()) < 20 and 
+            not any(threat in text.lower() for threat in threat_indicators) and
+            boundary_health > 0 and
+            sentiment == "supportive" and 
+            sentiment_confidence > 0.8
+        )
+
+    def _apply_score_modifiers(self, abuse_score, boundary_health, sentiment, explicit_abuse):
+        """Apply boundary health and sentiment modifiers to abuse score"""
+        if boundary_health > 0 and not explicit_abuse:
+            abuse_score = min(abuse_score, 35.0)
+        
+        if sentiment == "supportive" and not explicit_abuse:
+            abuse_score = min(abuse_score, 30.0)
+        
+        return abuse_score
+
+    def _analyze_enhanced_intent(self, text, fallacy_type, abuse_patterns):
+        """Enhanced intent analysis using fallacy and pattern context"""
+        # Get base intent from trained model
+        base_intent, base_confidence = self.model_manager.predict_intent(text)
+        
+        # Modify intent based on fallacy detection
+        intent_modifier = 0
+        likely_intent = base_intent
+        
+        # High-concern fallacies strongly suggest manipulation
+        if fallacy_type in ['Gaslighting', 'DARVO', 'Kafkatrapping']:
+            intent_modifier += 0.3
+            likely_intent = "manipulative"
+        elif fallacy_type in ['Appeal to Emotion', 'Ad Hominem']:
+            intent_modifier += 0.2
+            if base_intent == "neutral":
+                likely_intent = "manipulative"
+        
+        # Abuse patterns also modify intent
+        concerning_patterns = ['control', 'gaslighting', 'guilt tripping', 'blame shifting']
+        pattern_concern_count = sum(1 for pattern in abuse_patterns if pattern in concerning_patterns)
+        
+        if pattern_concern_count >= 2:
+            intent_modifier += 0.2
+            if likely_intent == "neutral":
+                likely_intent = "controlling"
+        
+        enhanced_confidence = min(base_confidence + intent_modifier, 1.0)
+        
+        return {
+            'primary_intent': likely_intent,
+            'model_predicted': base_intent,
+            'model_confidence': base_confidence,
+            'enhanced_confidence': enhanced_confidence,
+            'fallacy_influence': fallacy_type != 'No Fallacy',
+            'pattern_influence': pattern_concern_count > 0
+        }
+
+    def _calculate_enhanced_abuse_score(self, base_score, fallacy_type, fallacy_confidence, darvo_score):
+        """Calculate enhanced abuse score incorporating fallacy analysis"""
+        enhanced_score = base_score
+        
+        # Add fallacy impact based on severity
+        if fallacy_type != 'No Fallacy':
+            fallacy_weight = self.fallacy_weights.get(fallacy_type, 1.0)
+            fallacy_impact = fallacy_weight * fallacy_confidence * 12
+            enhanced_score += fallacy_impact
+        
+        # Special boost for DARVO combination
+        if fallacy_type == 'DARVO' and darvo_score > 0.7:
+            enhanced_score += 18
+        
+        # Gaslighting fallacy is particularly concerning
+        if fallacy_type == 'Gaslighting':
+            enhanced_score += 15
+        
+        return min(enhanced_score, 100.0)
+
+    def _calculate_abuser_indicators(self, base_analysis, fallacy_type, fallacy_confidence):
+        """Calculate indicators that suggest this sender might be an abuser"""
+        indicators = {
+            'high_abuse_score': base_analysis['abuse_score'] > 60,
+            'serious_fallacy': fallacy_type in ['Gaslighting', 'DARVO', 'Kafkatrapping', 'Appeal to Emotion'],
+            'high_darvo': base_analysis['darvo_score'] > 0.65,
+            'multiple_abuse_patterns': len(base_analysis['detected_patterns']) > 2,
+            'threatening_language': any(pattern in base_analysis['detected_patterns'] 
+                                      for pattern in ['veiled threats', 'stalking language', 'insults']),
+            'reality_distortion': fallacy_type == 'Gaslighting' or 'gaslighting' in base_analysis['detected_patterns'],
+            'victim_blaming': fallacy_type == 'DARVO' or base_analysis['darvo_score'] > 0.7
+        }
+        
+        indicator_count = sum(indicators.values())
+        risk_level = 'high' if indicator_count >= 4 else 'moderate' if indicator_count >= 2 else 'low'
+        
+        return {
+            'indicators': indicators,
+            'total_count': indicator_count,
+            'risk_level': risk_level
+        }
+
+    def _calculate_manipulation_score(self, abuse_patterns, fallacy_type):
+        """Calculate overall manipulation score"""
+        manipulation_patterns = [
+            'gaslighting', 'blame shifting', 'guilt tripping', 'false concern', 
+            'future faking', 'control'
+        ]
+        
+        pattern_score = sum(1 for pattern in abuse_patterns if pattern in manipulation_patterns)
+        fallacy_score = 2 if fallacy_type in ['Gaslighting', 'DARVO', 'Kafkatrapping'] else 1 if fallacy_type != 'No Fallacy' else 0
+        
+        return min(pattern_score + fallacy_score, 10)
+
+    def identify_primary_abuser(self, results_df):
+        """Enhanced primary abuser identification"""
+        logger.info("Identifying primary abuser with enhanced analysis...")
         
         sender_abuse_metrics = {}
         
         for sender in results_df['sender'].unique():
             sender_df = results_df[results_df['sender'] == sender]
             
-            if len(sender_df) < 3:  # Need minimum messages for reliable assessment
+            if len(sender_df) < 3:
                 continue
-                
-            # Calculate comprehensive abuse metrics
-            avg_abuse = sender_df['abuse_score'].mean()
-            max_abuse = sender_df['abuse_score'].max()
-            abusive_count = len(sender_df[sender_df['abuse_score'] >= 50])
-            abusive_pct = (abusive_count / len(sender_df)) * 100
-            
-            # Calculate pattern-weighted score
-            pattern_counts = Counter()
-            for patterns in sender_df['detected_patterns']:
-                pattern_counts.update(patterns)
             
-            # Calculate weighted pattern score
-            total_pattern_weight = 0
-            for pattern, count in pattern_counts.items():
-                weight = PATTERN_WEIGHTS.get(pattern, 1.0)  # Default weight of 1.0
-                total_pattern_weight += count * weight
-            
-            # Normalize by message count
-            weighted_pattern_score = total_pattern_weight / len(sender_df) if len(sender_df) > 0 else 0
-            
-            # DARVO score
-            avg_darvo = sender_df['darvo_score'].mean()
-            high_darvo_count = len(sender_df[sender_df['darvo_score'] >= 0.65])
-            
-            # Risk level distribution
-            high_risk_count = len(sender_df[sender_df['risk_level'].isin(['High', 'Critical'])])
-            high_risk_pct = (high_risk_count / len(sender_df)) * 100
-            
-            # Composite abuse score (weighted combination of factors)
-            composite_score = (
-                avg_abuse * 0.25 +
-                abusive_pct * 0.2 +
-                weighted_pattern_score * 15 +  # Heavily weight the pattern score
-                avg_darvo * 100 * 0.15 +
-                high_risk_pct * 0.1
-            )
-            
-            # Store detailed pattern information for reporting
-            pattern_details = [
-                {
-                    'pattern': pattern,
-                    'count': count,
-                    'weight': PATTERN_WEIGHTS.get(pattern, 1.0),
-                    'weighted_score': count * PATTERN_WEIGHTS.get(pattern, 1.0)
-                }
-                for pattern, count in pattern_counts.items()
-            ]
-            
-            # Sort patterns by weighted score
-            pattern_details.sort(key=lambda x: x['weighted_score'], reverse=True)
-            
-            sender_abuse_metrics[sender] = {
-                'message_count': len(sender_df),
-                'avg_abuse_score': avg_abuse,
-                'max_abuse_score': max_abuse,
-                'abusive_message_pct': abusive_pct,
-                'pattern_details': pattern_details,
-                'weighted_pattern_score': weighted_pattern_score,
-                'avg_darvo_score': avg_darvo,
-                'high_risk_pct': high_risk_pct,
-                'composite_score': composite_score
-            }
+            # Calculate comprehensive metrics
+            metrics = self._calculate_sender_metrics(sender_df)
+            sender_abuse_metrics[sender] = metrics
         
         if not sender_abuse_metrics:
             return None, sender_abuse_metrics
         
-        # Find primary abuser (highest composite score with minimum thresholds)
-        primary_abuser = None
-        max_composite = 0
-        
-        for sender, metrics in sender_abuse_metrics.items():
-            if (metrics['composite_score'] > max_composite and 
-                metrics['message_count'] >= 5):
-                max_composite = metrics['composite_score']
-                primary_abuser = sender
+        # Find primary abuser based on composite score
+        primary_abuser = max(
+            sender_abuse_metrics.keys(),
+            key=lambda x: sender_abuse_metrics[x]['composite_score']
+        )
         
         logger.info(f"Primary abuser identified: {primary_abuser}")
         return primary_abuser, sender_abuse_metrics
+
+    def _calculate_sender_metrics(self, sender_df):
+        """Calculate comprehensive metrics for a sender"""
+        # Basic abuse metrics
+        avg_abuse = sender_df['abuse_score'].mean()
+        max_abuse = sender_df['abuse_score'].max()
+        enhanced_avg = sender_df.get('enhanced_abuse_score', sender_df['abuse_score']).mean()
+        
+        abusive_count = len(sender_df[sender_df['abuse_score'] >= 50])
+        abusive_pct = (abusive_count / len(sender_df)) * 100
+        
+        # Fallacy metrics
+        fallacy_counts = Counter()
+        serious_fallacy_count = 0
+        
+        for _, row in sender_df.iterrows():
+            fallacy = row.get('fallacy_detected', 'No Fallacy')
+            if fallacy != 'No Fallacy':
+                fallacy_counts[fallacy] += 1
+                if fallacy in ['Gaslighting', 'DARVO', 'Kafkatrapping', 'Appeal to Emotion']:
+                    serious_fallacy_count += 1
+        
+        # Abuser indicators
+        total_abuser_indicators = 0
+        high_risk_messages = 0
+        
+        for _, row in sender_df.iterrows():
+            indicators = row.get('abuser_indicators', {})
+            if isinstance(indicators, dict):
+                total_abuser_indicators += indicators.get('total_count', 0)
+                if indicators.get('risk_level') == 'high':
+                    high_risk_messages += 1
+        
+        # DARVO and manipulation metrics
+        avg_darvo = sender_df['darvo_score'].mean()
+        high_darvo_count = len(sender_df[sender_df['darvo_score'] >= 0.65])
+        avg_manipulation = sender_df.get('manipulation_score', pd.Series([0] * len(sender_df))).mean()
+        
+        # Calculate composite score with enhanced weighting
+        composite_score = (
+            enhanced_avg * 0.25 +
+            abusive_pct * 0.15 +
+            (serious_fallacy_count / len(sender_df)) * 30 +
+            (total_abuser_indicators / len(sender_df)) * 20 +
+            avg_darvo * 100 * 0.1
+        )
+        
+        return {
+            'message_count': len(sender_df),
+            'avg_abuse_score': avg_abuse,
+            'enhanced_avg_abuse_score': enhanced_avg,
+            'max_abuse_score': max_abuse,
+            'abusive_message_pct': abusive_pct,
+            'fallacy_breakdown': dict(fallacy_counts),
+            'serious_fallacy_count': serious_fallacy_count,
+            'serious_fallacy_rate': serious_fallacy_count / len(sender_df),
+            'avg_darvo_score': avg_darvo,
+            'high_darvo_count': high_darvo_count,
+            'total_abuser_indicators': total_abuser_indicators,
+            'high_risk_messages': high_risk_messages,
+            'avg_manipulation_score': avg_manipulation,
+            'composite_score': composite_score,
+            'likely_abuser': composite_score > 50,
+            'confidence': 'high' if len(sender_df) >= 20 else 'medium' if len(sender_df) >= 10 else 'low'
+        }
+
     def analyze_chat_history(self, df):
-        """Analyze entire chat history"""
+        """Enhanced chat history analysis"""
         from utils import detect_escalation_patterns, generate_safety_plan, generate_professional_recommendations
         
         logger.info(f"Analyzing chat history with {len(df)} messages")
         
         try:
-            # Create results dataframe
             results_df = df.copy()
             
-            # Add analysis columns
-            results_df['abuse_score'] = 0.0
-            results_df['detected_patterns'] = [[] for _ in range(len(results_df))]
-            results_df['sentiment'] = "neutral"
-            results_df['darvo_score'] = 0.0
-            results_df['emotional_tone'] = "neutral"
-            results_df['boundary_health'] = "unknown"
-            results_df['risk_level'] = "Low"
+            # Initialize new columns for enhanced analysis
+            enhanced_columns = {
+                'abuse_score': 0.0,
+                'enhanced_abuse_score': 0.0,
+                'detected_patterns': [[] for _ in range(len(results_df))],
+                'sentiment': "neutral",
+                'darvo_score': 0.0,
+                'emotional_tone': "neutral",
+                'boundary_health': "unknown",
+                'risk_level': "Low",
+                'fallacy_detected': "No Fallacy",
+                'fallacy_confidence': 0.0,
+                'enhanced_intent': {},
+                'abuser_indicators': {},
+                'manipulation_score': 0
+            }
+            
+            for col, default_val in enhanced_columns.items():
+                results_df[col] = default_val
             
-            # Analyze each message
+            # Analyze each message with enhanced analysis
             for i, row in results_df.iterrows():
                 analysis = self.analyze_message(row['message'])
                 
-                # Update dataframe with analysis results
-                results_df.at[i, 'abuse_score'] = analysis['abuse_score']
-                results_df.at[i, 'detected_patterns'] = analysis['detected_patterns']
-                results_df.at[i, 'sentiment'] = analysis['sentiment']
-                results_df.at[i, 'darvo_score'] = analysis['darvo_score']
-                results_df.at[i, 'emotional_tone'] = analysis['emotional_tone']
-                results_df.at[i, 'boundary_health'] = analysis['boundary_assessment']['assessment']
-                results_df.at[i, 'risk_level'] = analysis['risk_level']
+                for col in enhanced_columns.keys():
+                    if col in analysis:
+                        results_df.at[i, col] = analysis[col]
             
-            # Calculate sender statistics
-            sender_stats = {}
-            for sender in results_df['sender'].unique():
-                sender_df = results_df[results_df['sender'] == sender]
-                
-                # Calculate key metrics
-                avg_abuse = sender_df['abuse_score'].mean()
-                max_abuse = sender_df['abuse_score'].max()
-                
-                # Get most common patterns
-                all_patterns = []
-                for patterns in sender_df['detected_patterns']:
-                    if patterns:
-                        all_patterns.extend(patterns)
-                
-                pattern_counts = Counter(all_patterns)
-                most_common = pattern_counts.most_common(5)  # Get top 5 patterns
-                
-                # Calculate percentage of abusive messages
-                abusive_count = len(sender_df[sender_df['abuse_score'] >= 50])
-                abusive_pct = (abusive_count / len(sender_df)) * 100 if len(sender_df) > 0 else 0
-                
-                # Calculate emotional tone distribution
-                tone_counts = Counter(sender_df['emotional_tone'])
-                most_common_tones = tone_counts.most_common(3)  # Get top 3 emotional tones
-                
-                # Calculate DARVO score statistics
-                avg_darvo = sender_df['darvo_score'].mean()
-                high_darvo_count = len(sender_df[sender_df['darvo_score'] >= 0.65])
-                high_darvo_pct = (high_darvo_count / len(sender_df)) * 100 if len(sender_df) > 0 else 0
-                
-                # Calculate risk level distribution
-                risk_counts = Counter(sender_df['risk_level'])
-                
-                # Store stats
-                sender_stats[sender] = {
-                    'message_count': len(sender_df),
-                    'avg_abuse_score': avg_abuse,
-                    'max_abuse_score': max_abuse,
-                    'abusive_message_count': abusive_count,
-                    'abusive_message_pct': abusive_pct,
-                    'common_patterns': most_common,
-                    'emotional_tones': most_common_tones,
-                    'avg_darvo_score': avg_darvo,
-                    'high_darvo_count': high_darvo_count,
-                    'high_darvo_pct': high_darvo_pct,
-                    'risk_levels': risk_counts
-                }
+            # Calculate enhanced sender statistics
+            sender_stats = self._calculate_enhanced_sender_stats(results_df)
             
-            # Identify primary abuser
+            # Identify primary abuser with enhanced method
             primary_abuser, sender_abuse_metrics = self.identify_primary_abuser(results_df)
             
             # Detect escalation patterns
             escalation_data = detect_escalation_patterns(results_df)
             
             # Determine overall risk level
-            if results_df['risk_level'].isin(['Critical']).any():
-                overall_risk = "Critical"
-            elif results_df['risk_level'].isin(['High']).any():
-                overall_risk = "High"
-            elif results_df['risk_level'].isin(['Moderate']).any():
-                overall_risk = "Moderate"
-            else:
-                overall_risk = "Low"
+            overall_risk = self._determine_overall_risk(results_df, sender_abuse_metrics)
             
-            # Generate safety plan
+            # Generate enhanced safety plan and recommendations
             all_patterns = []
             for patterns in results_df['detected_patterns']:
                 if patterns:
                     all_patterns.extend(patterns)
             
             safety_plan = generate_safety_plan(overall_risk, all_patterns, escalation_data)
-            
-            # Generate professional recommendations
             recommendations = generate_professional_recommendations(results_df, escalation_data, overall_risk)
             
-            # Prepare summary
+            # Enhanced summary with fallacy analysis
             summary = {
                 'message_count': len(results_df),
                 'date_range': {
@@ -416,7 +478,9 @@ class MessageAnalyzer:
                 'primary_abuser': primary_abuser,
                 'escalation_data': escalation_data,
                 'safety_plan': safety_plan,
-                'recommendations': recommendations
+                'recommendations': recommendations,
+                'fallacy_summary': self._create_fallacy_summary(results_df),
+                'enhanced_insights': self._create_enhanced_insights(results_df, primary_abuser)
             }
             
             return results_df, summary
@@ -424,17 +488,170 @@ class MessageAnalyzer:
         except Exception as e:
             logger.error(f"Error in analyze_chat_history: {e}")
             logger.error(traceback.format_exc())
-            return df, {
-                'message_count': len(df),
-                'date_range': {
-                    'start': df['timestamp'].min().strftime('%Y-%m-%d') if not df.empty else 'unknown',
-                    'end': df['timestamp'].max().strftime('%Y-%m-%d') if not df.empty else 'unknown'
-                },
-                'overall_risk_level': "Unknown",
-                'sender_stats': {},
-                'sender_abuse_metrics': {},
-                'primary_abuser': None,
-                'escalation_data': {},
-                'safety_plan': "Error generating safety plan.",
-                'recommendations': []
+            return df, self._get_error_summary(df)
+
+    def _calculate_enhanced_sender_stats(self, results_df):
+        """Calculate enhanced sender statistics including fallacy analysis"""
+        sender_stats = {}
+        
+        for sender in results_df['sender'].unique():
+            sender_df = results_df[results_df['sender'] == sender]
+            
+            # Basic stats
+            avg_abuse = sender_df['abuse_score'].mean()
+            enhanced_avg = sender_df['enhanced_abuse_score'].mean()
+            abusive_count = len(sender_df[sender_df['enhanced_abuse_score'] >= 50])
+            abusive_pct = (abusive_count / len(sender_df)) * 100 if len(sender_df) > 0 else 0
+            
+            # Pattern analysis
+            all_patterns = []
+            for patterns in sender_df['detected_patterns']:
+                if patterns:
+                    all_patterns.extend(patterns)
+            pattern_counts = Counter(all_patterns)
+            
+            # Fallacy analysis
+            fallacy_counts = Counter(sender_df['fallacy_detected'])
+            fallacy_counts.pop('No Fallacy', None)  # Remove 'No Fallacy' entries
+            
+            # Intent analysis
+            intent_counts = Counter()
+            for intent_data in sender_df['enhanced_intent']:
+                if isinstance(intent_data, dict):
+                    intent_counts[intent_data.get('primary_intent', 'unknown')] += 1
+            
+            sender_stats[sender] = {
+                'message_count': len(sender_df),
+                'avg_abuse_score': avg_abuse,
+                'enhanced_avg_abuse_score': enhanced_avg,
+                'abusive_message_count': abusive_count,
+                'abusive_message_pct': abusive_pct,
+                'common_patterns': pattern_counts.most_common(5),
+                'fallacy_usage': dict(fallacy_counts),
+                'intent_distribution': dict(intent_counts),
+                'avg_darvo_score': sender_df['darvo_score'].mean(),
+                'avg_manipulation_score': sender_df['manipulation_score'].mean()
             }
+        
+        return sender_stats
+
+    def _determine_overall_risk(self, results_df, sender_abuse_metrics):
+        """Determine overall risk level with enhanced criteria"""
+        if any(metrics.get('likely_abuser', False) for metrics in sender_abuse_metrics.values()):
+            return "Critical"
+        elif results_df['enhanced_abuse_score'].max() > 80:
+            return "High"
+        elif results_df['enhanced_abuse_score'].mean() > 40:
+            return "Moderate"
+        else:
+            return "Low"
+
+    def _create_fallacy_summary(self, results_df):
+        """Create summary of fallacy usage"""
+        fallacy_counts = Counter(results_df['fallacy_detected'])
+        fallacy_counts.pop('No Fallacy', None)
+        
+        total_messages = len(results_df)
+        messages_with_fallacies = total_messages - results_df[results_df['fallacy_detected'] == 'No Fallacy'].shape[0]
+        
+        return {
+            'total_fallacies_detected': sum(fallacy_counts.values()),
+            'messages_with_fallacies': messages_with_fallacies,
+            'fallacy_rate': round((messages_with_fallacies / total_messages) * 100, 1) if total_messages > 0 else 0,
+            'most_common_fallacies': dict(fallacy_counts.most_common(5)),
+            'serious_fallacies': {
+                fallacy: count for fallacy, count in fallacy_counts.items()
+                if fallacy in ['Gaslighting', 'DARVO', 'Kafkatrapping', 'Appeal to Emotion']
+            }
+        }
+
+    def _create_enhanced_insights(self, results_df, primary_abuser):
+        """Create enhanced insights from the analysis"""
+        insights = {
+            'key_findings': [],
+            'behavioral_patterns': {},
+            'risk_indicators': []
+        }
+        
+        # Key findings
+        if primary_abuser:
+            abuser_fallacies = results_df[results_df['sender'] == primary_abuser]['fallacy_detected']
+            serious_fallacies = [f for f in abuser_fallacies if f in ['Gaslighting', 'DARVO', 'Kafkatrapping']]
+            
+            if serious_fallacies:
+                insights['key_findings'].append(f"Primary abuser uses serious psychological manipulation tactics: {', '.join(set(serious_fallacies))}")
+        
+        # High manipulation scores
+        high_manipulation = results_df[results_df['manipulation_score'] >= 5]
+        if len(high_manipulation) > 0:
+            insights['key_findings'].append(f"{len(high_manipulation)} messages show high manipulation indicators")
+        
+        # Risk indicators
+        critical_patterns = ['stalking language', 'veiled threats', 'insults']
+        for pattern in critical_patterns:
+            pattern_count = sum(1 for patterns in results_df['detected_patterns'] if pattern in patterns)
+            if pattern_count > 0:
+                insights['risk_indicators'].append(f"{pattern_count} instances of {pattern}")
+        
+        return insights
+
+    def _get_empty_analysis(self):
+        """Return empty analysis for blank messages"""
+        return {
+            'abuse_score': 0.0,
+            'enhanced_abuse_score': 0.0,
+            'detected_patterns': [],
+            'matched_scores': [],
+            'sentiment': "neutral",
+            'sentiment_confidence': 0.5,
+            'stage': 1,
+            'darvo_score': 0.0,
+            'emotional_tone': "neutral",
+            'boundary_assessment': {'assessment': 'neutral', 'confidence': 0.5},
+            'risk_level': "Low",
+            'fallacy_detected': "No Fallacy",
+            'fallacy_confidence': 0.0,
+            'enhanced_intent': {'primary_intent': 'neutral'},
+            'abuser_indicators': {'total_count': 0, 'risk_level': 'low'},
+            'manipulation_score': 0
+        }
+
+    def _get_error_analysis(self):
+        """Return error analysis"""
+        return {
+            'abuse_score': 0.0,
+            'enhanced_abuse_score': 0.0,
+            'detected_patterns': [],
+            'matched_scores': [],
+            'sentiment': "error",
+            'sentiment_confidence': 0.0,
+            'stage': 1,
+            'darvo_score': 0.0,
+            'emotional_tone': "error",
+            'boundary_assessment': {'assessment': 'error', 'confidence': 0.0},
+            'risk_level': "Unknown",
+            'fallacy_detected': "No Fallacy",
+            'fallacy_confidence': 0.0,
+            'enhanced_intent': {'primary_intent': 'error'},
+            'abuser_indicators': {'total_count': 0, 'risk_level': 'unknown'},
+            'manipulation_score': 0
+        }
+
+    def _get_error_summary(self, df):
+        """Return error summary"""
+        return {
+            'message_count': len(df),
+            'date_range': {
+                'start': df['timestamp'].min().strftime('%Y-%m-%d') if not df.empty else 'unknown',
+                'end': df['timestamp'].max().strftime('%Y-%m-%d') if not df.empty else 'unknown'
+            },
+            'overall_risk_level': "Unknown",
+            'sender_stats': {},
+            'sender_abuse_metrics': {},
+            'primary_abuser': None,
+            'escalation_data': {},
+            'safety_plan': "Error generating safety plan.",
+            'recommendations': [],
+            'fallacy_summary': {},
+            'enhanced_insights': {}
+        }