Add extraction and relevance evaluators for condition extraction and retrieval relevance analysis

- Implemented `extraction_evaluator.py` to evaluate condition extraction success rates using the `UserPromptProcessor`.
- Added functionality to parse queries from a file, evaluate extractions, and calculate statistics.
- Created methods to save extraction statistics and detailed results in JSON format.
- Implemented `relevance_evaluator.py` to assess retrieval relevance using cosine similarity scores.
- Included methods for parsing queries, evaluating relevance, and generating statistics.
- Both evaluators support independent execution and provide detailed output for analysis.

evaluation/latency_evaluator.py

@@ -1,10 +1,21 @@
 #!/usr/bin/env python3
 """
-OnCall.ai System - Latency Evaluator (Single Query Test Mode)
-============================================================
+OnCall.ai System - Comprehensive Evaluator (Metrics 1-6)
+========================================================
 
-Test latency for individual queries to avoid rate limits.
-Based on existing system flow: app.py -> user_prompt.py -> retrieval.py -> generation.py
+Single execution to collect all metrics 1-6 data from app.py pipeline:
+
+RETRIEVAL METRICS (Only available for RAG systems):
+1. Total Latency (總處理時長) - Complete pipeline timing
+2. Condition Extraction Success Rate (條件抽取成功率) - user_prompt.py success  
+3. Retrieval Relevance (檢索相關性) - cosine similarity from retrieval.py
+4. Retrieval Coverage (檢索覆蓋率) - advice utilization of retrieved content
+
+LLM EVALUATION METRICS (Available for all systems):
+5. Clinical Actionability (臨床可操作性) - Third-party LLM evaluation  
+6. Clinical Evidence Quality (臨床證據品質) - Third-party LLM evaluation
+
+Note: This evaluator focuses on metrics 1-4. Metrics 5-6 require separate LLM evaluation.
 
 Author: YanBo Chen  
 Date: 2025-08-04
@@ -14,7 +25,7 @@ import time
 import json
 import os
 import sys
-from typing import Dict, List, Any
+from typing import Dict, List, Any, Set
 from datetime import datetime
 from pathlib import Path
 import re
@@ -37,12 +48,12 @@ except ImportError as e:
     sys.exit(1)
 
 
-class LatencyEvaluator:
-    """Pure latency measurement and medical advice output recording - no visualization"""
+class ComprehensiveEvaluator:
+    """Comprehensive evaluator for metrics 1-4 - single execution approach"""
     
     def __init__(self):
-        """Initialize existing system components"""
-        print("🔧 Initializing Latency Evaluator...")
+        """Initialize system components (identical to app.py)"""
+        print("🔧 Initializing Comprehensive Evaluator...")
         
         # Initialize existing system components (same as app.py)
         self.llm_client = llm_Med42_70BClient()
@@ -53,66 +64,137 @@ class LatencyEvaluator:
         )
         self.medical_generator = MedicalAdviceGenerator(llm_client=self.llm_client)
         
-        # Results accumulation for summary statistics
-        self.accumulated_results = {
-            "diagnosis": [],
-            "treatment": [],
-            "mixed": []
+        # Results accumulation for all metrics
+        self.comprehensive_results = []
+        self.medical_outputs = []
+        
+        print("✅ Comprehensive Evaluator initialization complete")
+    
+    def extract_medical_keywords(self, text: str) -> Set[str]:
+        """Extract medical keywords for coverage analysis"""
+        if not text:
+            return set()
+        
+        medical_keywords = set()
+        text_lower = text.lower()
+        
+        # Medical terminology patterns
+        patterns = [
+            r'\b[a-z]+(?:osis|itis|pathy|emia|uria|gram|scopy)\b',  # Medical suffixes
+            r'\b(?:cardio|neuro|pulmo|gastro|hepato|nephro)[a-z]+\b',  # Medical prefixes
+            r'\b(?:diagnosis|treatment|therapy|intervention|management)\b',  # Medical actions
+            r'\b(?:patient|symptom|condition|disease|disorder|syndrome)\b',  # Medical entities
+            r'\b(?:acute|chronic|severe|mild|moderate|emergency)\b',  # Medical descriptors
+            r'\b[a-z]+(?:al|ic|ous|ive)\s+(?:pain|failure|infection|injury)\b',  # Compound terms
+            r'\b(?:ecg|ekg|ct|mri|x-ray|ultrasound|biopsy)\b',  # Medical procedures
+            r'\b\d+\s*(?:mg|ml|units|hours|days|minutes)\b',  # Dosages and timeframes
+        ]
+        
+        for pattern in patterns:
+            matches = re.findall(pattern, text_lower)
+            medical_keywords.update(match.strip() for match in matches)
+        
+        # Additional common medical terms
+        common_medical_terms = [
+            'blood', 'pressure', 'heart', 'chest', 'pain', 'stroke', 'seizure',
+            'emergency', 'hospital', 'monitor', 'assess', 'evaluate', 'immediate',
+            'protocol', 'guideline', 'recommendation', 'risk', 'factor'
+        ]
+        
+        for term in common_medical_terms:
+            if term in text_lower:
+                medical_keywords.add(term)
+        
+        # Filter out very short terms and common words
+        filtered_keywords = {
+            kw for kw in medical_keywords 
+            if len(kw) > 2 and kw not in ['the', 'and', 'for', 'with', 'are', 'can', 'may']
         }
         
-        # Medical advice outputs for model comparison
-        self.medical_outputs = []
+        return filtered_keywords
+    
+    def calculate_coverage_metrics(self, generated_advice: str, retrieval_results: List[Dict]) -> Dict[str, Any]:
+        """Calculate coverage metrics from generated advice and retrieval results"""
+        if not generated_advice or not retrieval_results:
+            return {
+                "coverage_score": 0.0,
+                "matched_keywords": [],
+                "advice_keywords": [],
+                "source_keywords": [],
+                "coverage_percentage": 0.0,
+                "meets_threshold": False
+            }
+        
+        # Extract keywords from generated advice
+        advice_keywords = self.extract_medical_keywords(generated_advice)
+        
+        # Extract keywords from all retrieved documents
+        all_source_keywords = set()
+        for doc in retrieval_results:
+            doc_content = doc.get('content', '') or doc.get('text', '')
+            doc_keywords = self.extract_medical_keywords(doc_content)
+            all_source_keywords.update(doc_keywords)
+        
+        # Calculate coverage
+        matched_keywords = advice_keywords.intersection(all_source_keywords)
+        coverage_score = len(matched_keywords) / len(all_source_keywords) if all_source_keywords else 0.0
         
-        print("✅ Latency Evaluator initialization complete")
+        return {
+            "coverage_score": coverage_score,
+            "matched_keywords": list(matched_keywords),
+            "advice_keywords": list(advice_keywords),
+            "source_keywords": list(all_source_keywords),
+            "advice_keywords_count": len(advice_keywords),
+            "source_keywords_count": len(all_source_keywords),
+            "matched_keywords_count": len(matched_keywords),
+            "coverage_percentage": coverage_score * 100,
+            "meets_threshold": coverage_score >= 0.6
+        }
     
-    def measure_single_query_latency(self, query: str, category: str = "unknown") -> Dict[str, Any]:
+    def evaluate_single_query_comprehensive(self, query: str, category: str = "unknown") -> Dict[str, Any]:
         """
-        Measure complete processing time for a single query
+        Comprehensive evaluation for single query - collects all metrics 1-4 data
         
-        Replicates app.py's process_medical_query flow with timing focus
+        Replicates app.py's process_medical_query pipeline exactly
         
         Args:
             query: Medical query to test
             category: Query category (diagnosis/treatment/mixed)
         """
-        print(f"⏱️ Measuring query latency: {query[:50]}...")
+        print(f"🔍 Comprehensive evaluation: {query[:50]}...")
         print(f"📋 Category: {category}")
         
         overall_start = time.time()
         timing_details = {}
         
         try:
-            # STEP 1: Condition extraction (user_prompt.py)
+            # STEP 1: Query Processing and Condition Extraction (identical to app.py)
             step1_start = time.time()
             condition_result = self.user_prompt_processor.extract_condition_keywords(query)
-            timing_details['step1_condition_extraction'] = time.time() - step1_start
+            step1_time = time.time() - step1_start
+            timing_details['step1_condition_extraction'] = step1_time
             
-            print(f"   Step 1 - Condition extraction: {timing_details['step1_condition_extraction']:.3f}s")
+            print(f"   Step 1 - Condition extraction: {step1_time:.3f}s")
             print(f"   Extracted condition: {condition_result.get('condition', 'None')}")
             
             # Check if valid medical query
             if condition_result.get('query_status') in ['invalid_query', 'non_medical']:
                 total_time = time.time() - overall_start
-                print(f"   ⚠️ Non-medical query detected")
-                return {
-                    "query": query,
-                    "category": category,
-                    "total_latency": total_time,
-                    "timing_details": timing_details,
-                    "status": "non_medical",
-                    "condition_result": condition_result,
-                    "success": False,
-                    "timestamp": datetime.now().isoformat()
-                }
+                return self._create_failed_result(query, category, total_time, timing_details, 
+                                                "non_medical", condition_result)
             
-            # STEP 2: User confirmation (simulate auto-confirmation)
+            # STEP 2: User Confirmation (simulate auto-confirmation)
             step2_start = time.time()
             confirmation = self.user_prompt_processor.handle_user_confirmation(condition_result)
-            timing_details['step2_confirmation'] = time.time() - step2_start
+            step2_time = time.time() - step2_start
+            timing_details['step2_confirmation'] = step2_time
             
-            print(f"   Step 2 - User confirmation: {timing_details['step2_confirmation']:.3f}s")
+            if not condition_result.get('condition'):
+                total_time = time.time() - overall_start
+                return self._create_failed_result(query, category, total_time, timing_details,
+                                                "no_condition", condition_result)
             
-            # STEP 3: Retrieve relevant guidelines (retrieval.py)
+            # STEP 3: Medical Guidelines Retrieval (identical to app.py)
             step3_start = time.time()
             
             search_query = f"{condition_result.get('emergency_keywords', '')} {condition_result.get('treatment_keywords', '')}".strip()
@@ -120,12 +202,13 @@ class LatencyEvaluator:
                 search_query = condition_result.get('condition', query)
             
             retrieval_results = self.retrieval_system.search(search_query, top_k=5)
-            timing_details['step3_retrieval'] = time.time() - step3_start
+            step3_time = time.time() - step3_start
+            timing_details['step3_retrieval'] = step3_time
             
-            retrieved_count = len(retrieval_results.get('processed_results', []))
-            print(f"   Step 3 - Retrieval: {timing_details['step3_retrieval']:.3f}s ({retrieved_count} results)")
+            processed_results = retrieval_results.get('processed_results', [])
+            print(f"   Step 3 - Retrieval: {step3_time:.3f}s ({len(processed_results)} results)")
             
-            # STEP 4: Generate medical advice (generation.py)
+            # STEP 4: Medical Advice Generation (identical to app.py)
             step4_start = time.time()
             
             intention = self._detect_query_intention(query)
@@ -134,68 +217,199 @@ class LatencyEvaluator:
                 retrieval_results=retrieval_results,
                 intention=intention
             )
-            timing_details['step4_generation'] = time.time() - step4_start
+            step4_time = time.time() - step4_start
+            timing_details['step4_generation'] = step4_time
+            
+            generated_advice = medical_advice_result.get('medical_advice', '')
+            confidence_score = medical_advice_result.get('confidence_score', 0.0)
             
-            print(f"   Step 4 - Generation: {timing_details['step4_generation']:.3f}s")
+            print(f"   Step 4 - Generation: {step4_time:.3f}s")
             
             total_time = time.time() - overall_start
             
-            # Extract medical advice output for future model comparison
-            medical_advice_text = medical_advice_result.get('medical_advice', '')
-            confidence_score = medical_advice_result.get('confidence_score', 0.0)
+            # METRIC 2: Condition Extraction Analysis
+            extraction_success = (
+                condition_result.get('condition') and
+                condition_result.get('condition') != "unknown" and
+                condition_result.get('query_status') not in ['invalid_query', 'non_medical']
+            )
+            
+            extraction_metrics = {
+                "extraction_success": extraction_success,
+                "extracted_condition": condition_result.get('condition'),
+                "query_status": condition_result.get('query_status'),
+                "emergency_keywords": condition_result.get('emergency_keywords', []),
+                "treatment_keywords": condition_result.get('treatment_keywords', []),
+                "fallback_level": condition_result.get('fallback_level', 'unknown'),
+                "extraction_time": step1_time
+            }
+            
+            # METRIC 3: Retrieval Relevance Analysis
+            if processed_results:
+                similarity_scores = []
+                for doc_result in processed_results:
+                    similarity = (
+                        doc_result.get('distance', 0.0) or 
+                        doc_result.get('similarity_score', 0.0) or
+                        doc_result.get('score', 0.0)
+                    )
+                    similarity_scores.append(similarity)
+                
+                average_relevance = sum(similarity_scores) / len(similarity_scores)
+                high_relevance_count = sum(1 for score in similarity_scores if score >= 0.2)
+                
+                relevance_metrics = {
+                    "average_relevance": average_relevance,
+                    "max_relevance": max(similarity_scores),
+                    "min_relevance": min(similarity_scores),
+                    "similarity_scores": similarity_scores,
+                    "high_relevance_count": high_relevance_count,
+                    "high_relevance_ratio": high_relevance_count / len(similarity_scores),
+                    "retrieved_count": len(processed_results),
+                    "meets_threshold": average_relevance >= 0.2,
+                    "retrieval_time": step3_time
+                }
+            else:
+                relevance_metrics = {
+                    "average_relevance": 0.0,
+                    "max_relevance": 0.0,
+                    "min_relevance": 0.0,
+                    "similarity_scores": [],
+                    "high_relevance_count": 0,
+                    "high_relevance_ratio": 0.0,
+                    "retrieved_count": 0,
+                    "meets_threshold": False,
+                    "retrieval_time": step3_time
+                }
+            
+            # METRIC 4: Retrieval Coverage Analysis
+            coverage_metrics = self.calculate_coverage_metrics(generated_advice, processed_results)
+            coverage_metrics["generation_time"] = step4_time
             
-            result = {
+            # Create comprehensive result
+            comprehensive_result = {
                 "query": query,
                 "category": category,
-                "total_latency": total_time,
-                "timing_details": timing_details,
-                "condition_result": condition_result,
-                "retrieval_results": retrieval_results,
-                "medical_advice_result": medical_advice_result,
-                "status": "success",
-                "success": True,
+                
+                # Metric 1: Total Latency - Complete pipeline processing time
+                "latency_metrics": {
+                    "total_latency": total_time,
+                    "timing_details": timing_details,
+                    "meets_target": total_time <= 30.0
+                },
+                
+                # Metric 2: Condition Extraction - Success rate from user_prompt.py
+                "extraction_metrics": extraction_metrics,
+                
+                # Metric 3: Retrieval Relevance - Cosine similarity from retrieval.py  
+                "relevance_metrics": relevance_metrics,
+                
+                # Metric 4: Retrieval Coverage - Advice utilization of retrieved content
+                "coverage_metrics": coverage_metrics,
+                
+                # Complete pipeline data (for debugging and detailed analysis)
+                "pipeline_data": {
+                    "condition_result": condition_result,
+                    "retrieval_results": retrieval_results,
+                    "medical_advice_result": medical_advice_result,
+                    "search_query": search_query,
+                    "intention": intention
+                },
+                
+                "overall_success": True,
                 "timestamp": datetime.now().isoformat()
             }
             
-            # Store medical output separately for model comparison
+            # Store result
+            self.comprehensive_results.append(comprehensive_result)
+            
+            # Store medical output for model comparison
             medical_output = {
                 "query": query,
                 "category": category,
-                "medical_advice": medical_advice_text,
+                "medical_advice": generated_advice,
                 "confidence_score": confidence_score,
                 "query_id": f"{category}_query",
                 "processing_time": total_time,
                 "timestamp": datetime.now().isoformat()
             }
-            
             self.medical_outputs.append(medical_output)
             
-            print(f"✅ Query completed successfully in {total_time:.2f}s")
-            print(f"📝 Medical advice recorded ({len(medical_advice_text)} characters)")
+            print(f"✅ Comprehensive evaluation completed in {total_time:.2f}s")
+            print(f"   📊 Metrics: Latency={total_time:.2f}s, Extraction={'✅' if extraction_success else '❌'}, "
+                  f"Relevance={average_relevance:.3f}, Coverage={coverage_metrics['coverage_score']:.3f}")
             
-            return result
+            return comprehensive_result
             
         except Exception as e:
             total_time = time.time() - overall_start
-            print(f"❌ Query failed after {total_time:.2f}s: {e}")
+            print(f"❌ Comprehensive evaluation failed after {total_time:.2f}s: {e}")
             
-            return {
-                "query": query,
-                "category": category,
+            return self._create_failed_result(query, category, total_time, timing_details, "error", None, str(e))
+    
+    def _create_failed_result(self, query: str, category: str, total_time: float, 
+                            timing_details: Dict, status: str, condition_result: Dict = None, 
+                            error: str = None) -> Dict[str, Any]:
+        """Create standardized failed result"""
+        failed_result = {
+            "query": query,
+            "category": category,
+            
+            # Metric 1: Total Latency - Always measurable even on failure
+            "latency_metrics": {
                 "total_latency": total_time,
                 "timing_details": timing_details,
-                "error": str(e),
-                "status": "error",
-                "success": False,
-                "timestamp": datetime.now().isoformat()
-            }
+                "meets_target": total_time <= 30.0
+            },
+            
+            # Metric 2: Condition Extraction - Partial data may be available before failure
+            "extraction_metrics": {
+                "extraction_success": False,
+                "extracted_condition": condition_result.get('condition') if condition_result else None,
+                "query_status": condition_result.get('query_status') if condition_result else status,
+                "extraction_time": timing_details.get('step1_condition_extraction', 0.0)
+            },
+            
+            # Metric 3: Retrieval Relevance - Failed due to pipeline failure
+            "relevance_metrics": {
+                "average_relevance": 0.0,
+                "retrieved_count": 0,
+                "meets_threshold": False,
+                "retrieval_time": timing_details.get('step3_retrieval', 0.0)
+            },
+            
+            # Metric 4: Retrieval Coverage - Failed due to pipeline failure  
+            "coverage_metrics": {
+                "coverage_score": 0.0,
+                "meets_threshold": False,
+                "generation_time": timing_details.get('step4_generation', 0.0)
+            },
+            
+            # Note: Metrics 5-6 (Clinical Actionability & Evidence Quality) 
+            # are not collected here - they require separate LLM evaluation
+            # using the medical_outputs saved by this evaluator
+            
+            "overall_success": False,
+            "status": status,
+            "error": error,
+            "timestamp": datetime.now().isoformat()
+        }
+        
+        self.comprehensive_results.append(failed_result)
+        return failed_result
     
-    def test_individual_queries_from_file(self, filepath: str) -> Dict[str, List[Dict]]:
-        """
-        Parse queries from file and return them for individual testing
+    def _detect_query_intention(self, query: str) -> str:
+        """Simplified query intention detection (from app.py)"""
+        query_lower = query.lower()
         
-        Returns categorized queries for separate testing
-        """
+        if any(word in query_lower for word in ['diagnos', 'differential', 'possible', 'causes']):
+            return 'diagnosis'
+        elif any(word in query_lower for word in ['treat', 'manage', 'therapy', 'intervention']):
+            return 'treatment'
+        else:
+            return 'mixed'    
+    def parse_queries_from_file(self, filepath: str) -> Dict[str, List[Dict]]:
+        """Parse queries from file with category labels"""
         print(f"📁 Reading queries from file: {filepath}")
         
         try:
@@ -237,8 +451,6 @@ class LatencyEvaluator:
             print(f"📋 Parsed queries by category:")
             for category, category_queries in queries_by_category.items():
                 print(f"  {category.capitalize()}: {len(category_queries)} queries")
-                for i, query_info in enumerate(category_queries):
-                    print(f"    {i+1}. {query_info['text'][:60]}...")
             
             return queries_by_category
             
@@ -246,23 +458,225 @@ class LatencyEvaluator:
             print(f"❌ Failed to read file: {e}")
             return {"error": f"Failed to read file: {e}"}
     
-    def _detect_query_intention(self, query: str) -> str:
-        """Simplified query intention detection (from app.py)"""
-        query_lower = query.lower()
+    def calculate_metric_statistics(self, metric_name: str) -> Dict[str, Any]:
+        """Calculate statistics for a specific metric across all results"""
+        category_stats = {}
+        all_successful_results = []
         
-        if any(word in query_lower for word in ['diagnos', 'differential', 'possible', 'causes']):
-            return 'diagnosis'
-        elif any(word in query_lower for word in ['treat', 'manage', 'therapy', 'intervention']):
-            return 'treatment'
-        else:
-            return 'mixed'
+        # Group results by category
+        results_by_category = {
+            "diagnosis": [],
+            "treatment": [],
+            "mixed": []
+        }
+        
+        for result in self.comprehensive_results:
+            category = result.get('category', 'unknown')
+            if category in results_by_category:
+                results_by_category[category].append(result)
+                if result.get('overall_success'):
+                    all_successful_results.append(result)
+        
+        # Calculate statistics for each category based on metric type
+        for category, results in results_by_category.items():
+            successful_results = [r for r in results if r.get('overall_success')]
+            
+            if metric_name == "latency":
+                if successful_results:
+                    latencies = [r['latency_metrics']['total_latency'] for r in successful_results]
+                    category_stats[category] = {
+                        "average_latency": sum(latencies) / len(latencies),
+                        "std_deviation": self._calculate_std(latencies),
+                        "min_latency": min(latencies),
+                        "max_latency": max(latencies),
+                        "query_count": len(latencies),
+                        "target_compliance": sum(1 for lat in latencies if lat <= 30.0) / len(latencies),
+                        "individual_latencies": latencies
+                    }
+                else:
+                    category_stats[category] = self._get_empty_latency_stats()
+            
+            elif metric_name == "extraction":
+                extraction_successes = [r['extraction_metrics']['extraction_success'] for r in results]
+                successful_extractions = sum(extraction_successes)
+                
+                category_stats[category] = {
+                    "success_rate": successful_extractions / len(results) if results else 0.0,
+                    "successful_count": successful_extractions,
+                    "total_count": len(results),
+                    "average_extraction_time": sum(r['extraction_metrics']['extraction_time'] for r in results) / len(results) if results else 0.0,
+                    "meets_threshold": (successful_extractions / len(results)) >= 0.8 if results else False
+                }
+            
+            elif metric_name == "relevance":
+                if successful_results:
+                    relevance_scores = [r['relevance_metrics']['average_relevance'] for r in successful_results]
+                    category_stats[category] = {
+                        "average_relevance": sum(relevance_scores) / len(relevance_scores),
+                        "max_relevance": max(relevance_scores),
+                        "min_relevance": min(relevance_scores),
+                        "successful_retrievals": len(successful_results),
+                        "total_queries": len(results),
+                        "meets_threshold": (sum(relevance_scores) / len(relevance_scores)) >= 0.2,
+                        "individual_relevance_scores": relevance_scores
+                    }
+                else:
+                    category_stats[category] = self._get_empty_relevance_stats(len(results))
+            
+            elif metric_name == "coverage":
+                if successful_results:
+                    coverage_scores = [r['coverage_metrics']['coverage_score'] for r in successful_results]
+                    category_stats[category] = {
+                        "average_coverage": sum(coverage_scores) / len(coverage_scores),
+                        "max_coverage": max(coverage_scores),
+                        "min_coverage": min(coverage_scores),
+                        "successful_evaluations": len(successful_results),
+                        "total_queries": len(results),
+                        "meets_threshold": (sum(coverage_scores) / len(coverage_scores)) >= 0.6,
+                        "individual_coverage_scores": coverage_scores
+                    }
+                else:
+                    category_stats[category] = self._get_empty_coverage_stats(len(results))
+        
+        # Calculate overall statistics
+        overall_stats = self._calculate_overall_stats(metric_name, all_successful_results)
+        
+        return {
+            "category_results": category_stats,
+            "overall_results": overall_stats,
+            "timestamp": datetime.now().isoformat()
+        }
+    
+    def _calculate_std(self, values: List[float]) -> float:
+        """Calculate standard deviation"""
+        if len(values) < 2:
+            return 0.0
+        
+        mean = sum(values) / len(values)
+        variance = sum((x - mean) ** 2 for x in values) / len(values)
+        return variance ** 0.5
     
-    def save_single_result(self, result: Dict[str, Any], filename: str = None) -> str:
-        """Save single query evaluation result"""
+    def _get_empty_latency_stats(self) -> Dict[str, Any]:
+        """Return empty latency statistics"""
+        return {
+            "average_latency": 0.0,
+            "std_deviation": 0.0,
+            "min_latency": 0.0,
+            "max_latency": 0.0,
+            "query_count": 0,
+            "target_compliance": 0.0,
+            "individual_latencies": []
+        }
+    
+    def _get_empty_relevance_stats(self, total_queries: int) -> Dict[str, Any]:
+        """Return empty relevance statistics"""
+        return {
+            "average_relevance": 0.0,
+            "max_relevance": 0.0,
+            "min_relevance": 0.0,
+            "successful_retrievals": 0,
+            "total_queries": total_queries,
+            "meets_threshold": False,
+            "individual_relevance_scores": []
+        }
+    
+    def _get_empty_coverage_stats(self, total_queries: int) -> Dict[str, Any]:
+        """Return empty coverage statistics"""
+        return {
+            "average_coverage": 0.0,
+            "max_coverage": 0.0,
+            "min_coverage": 0.0,
+            "successful_evaluations": 0,
+            "total_queries": total_queries,
+            "meets_threshold": False,
+            "individual_coverage_scores": []
+        }
+    
+    def _calculate_overall_stats(self, metric_name: str, all_successful_results: List[Dict]) -> Dict[str, Any]:
+        """Calculate overall statistics for a specific metric"""
+        total_queries = len(self.comprehensive_results)
+        
+        if metric_name == "latency" and all_successful_results:
+            latencies = [r['latency_metrics']['total_latency'] for r in all_successful_results]
+            return {
+                "average_latency": sum(latencies) / len(latencies),
+                "std_deviation": self._calculate_std(latencies),
+                "min_latency": min(latencies),
+                "max_latency": max(latencies),
+                "successful_queries": len(all_successful_results),
+                "total_queries": total_queries,
+                "target_compliance": sum(1 for lat in latencies if lat <= 30.0) / len(latencies)
+            }
+        
+        elif metric_name == "extraction":
+            all_extractions = [r['extraction_metrics']['extraction_success'] for r in self.comprehensive_results]
+            successful_extractions = sum(all_extractions)
+            return {
+                "success_rate": successful_extractions / len(all_extractions) if all_extractions else 0.0,
+                "successful_count": successful_extractions,
+                "total_count": len(all_extractions),
+                "target_compliance": (successful_extractions / len(all_extractions)) >= 0.8 if all_extractions else False
+            }
+        
+        elif metric_name == "relevance" and all_successful_results:
+            relevance_scores = [r['relevance_metrics']['average_relevance'] for r in all_successful_results]
+            return {
+                "average_relevance": sum(relevance_scores) / len(relevance_scores),
+                "max_relevance": max(relevance_scores),
+                "min_relevance": min(relevance_scores),
+                "successful_queries": len(all_successful_results),
+                "total_queries": total_queries,
+                "meets_threshold": (sum(relevance_scores) / len(relevance_scores)) >= 0.2,
+                "target_compliance": (sum(relevance_scores) / len(relevance_scores)) >= 0.25
+            }
+        
+        elif metric_name == "coverage" and all_successful_results:
+            coverage_scores = [r['coverage_metrics']['coverage_score'] for r in all_successful_results]
+            return {
+                "average_coverage": sum(coverage_scores) / len(coverage_scores),
+                "max_coverage": max(coverage_scores),
+                "min_coverage": min(coverage_scores),
+                "successful_queries": len(all_successful_results),
+                "total_queries": total_queries,
+                "meets_threshold": (sum(coverage_scores) / len(coverage_scores)) >= 0.6
+            }
+        
+        # Return empty stats for failed cases
+        return {
+            "average_value": 0.0,
+            "successful_queries": len(all_successful_results),
+            "total_queries": total_queries,
+            "meets_threshold": False
+        }    
+    def save_all_metric_statistics(self) -> Dict[str, str]:
+        """Save separate statistics files for each metric"""
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        saved_files = {}
+        
+        # Save statistics for each metric
+        for metric_name in ["latency", "extraction", "relevance", "coverage"]:
+            stats = self.calculate_metric_statistics(metric_name)
+            filename = f"{metric_name}_statistics_{timestamp}.json"
+            filepath = results_dir / filename
+            
+            with open(filepath, 'w', encoding='utf-8') as f:
+                json.dump(stats, f, indent=2, ensure_ascii=False)
+            
+            saved_files[metric_name] = str(filepath)
+            print(f"📊 {metric_name.capitalize()} statistics saved to: {filepath}")
+        
+        return saved_files
+    
+    def save_medical_outputs(self, filename: str = None) -> str:
+        """Save medical advice outputs for model comparison"""
         if filename is None:
             timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-            category = result.get('category', 'unknown')
-            filename = f"latency_{category}_{timestamp}.json"
+            filename = f"medical_outputs_{timestamp}.json"
         
         # Ensure results directory exists
         results_dir = Path(__file__).parent / "results"
@@ -270,18 +684,59 @@ class LatencyEvaluator:
         
         filepath = results_dir / filename
         
+        # Create comprehensive output data
+        output_data = {
+            "evaluation_metadata": {
+                "total_outputs": len(self.medical_outputs),
+                "categories": list(set(output['category'] for output in self.medical_outputs)),
+                "timestamp": datetime.now().isoformat(),
+                "model_type": "Med42-70B_RAG_enhanced"  # For future comparison
+            },
+            "medical_outputs": self.medical_outputs
+        }
+        
         with open(filepath, 'w', encoding='utf-8') as f:
-            json.dump(result, f, indent=2, ensure_ascii=False)
+            json.dump(output_data, f, indent=2, ensure_ascii=False)
         
-        print(f"💾 Result saved to: {filepath}")
+        print(f"📝 Medical outputs saved to: {filepath}")
+        return str(filepath)
+    
+    def save_comprehensive_details(self, filename: str = None) -> str:
+        """Save comprehensive detailed results"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"comprehensive_details_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        # Create comprehensive evaluation data
+        comprehensive_data = {
+            "evaluation_metadata": {
+                "total_queries": len(self.comprehensive_results),
+                "successful_queries": len([r for r in self.comprehensive_results if r.get('overall_success')]),
+                "timestamp": datetime.now().isoformat(),
+                "evaluator_type": "comprehensive_metrics_1_to_4",
+                "metrics_evaluated": ["latency", "extraction", "relevance", "coverage"]
+            },
+            "comprehensive_results": self.comprehensive_results
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(comprehensive_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"📋 Comprehensive details saved to: {filepath}")
         return str(filepath)
 
 
 # Independent execution interface
 if __name__ == "__main__":
-    """Independent test interface for single queries"""
+    """Independent comprehensive evaluation interface"""
     
-    print("🚀 OnCall.ai Latency Evaluator - Single Query Test Mode")
+    print("🚀 OnCall.ai Comprehensive Evaluator - Metrics 1-4 in Single Run")
     
     if len(sys.argv) > 1:
         query_file = sys.argv[1]
@@ -295,18 +750,18 @@ if __name__ == "__main__":
         sys.exit(1)
     
     # Initialize evaluator
-    evaluator = LatencyEvaluator()
+    evaluator = ComprehensiveEvaluator()
     
     # Parse queries from file
-    queries_by_category = evaluator.test_individual_queries_from_file(str(query_file))
+    queries_by_category = evaluator.parse_queries_from_file(str(query_file))
     
     if "error" in queries_by_category:
         print(f"❌ Failed to parse queries: {queries_by_category['error']}")
         sys.exit(1)
     
-    # Test each category individually
-    print(f"\n🧪 Individual Query Testing Mode with Result Accumulation")
-    print(f"📝 Test each query separately to avoid rate limits")
+    # Test each query comprehensively
+    print(f"\n🧪 Comprehensive Evaluation - All Metrics in Single Run")
+    print(f"📊 Collecting metrics 1-4 from single app.py pipeline execution")
     
     for category, queries in queries_by_category.items():
         if not queries:
@@ -319,178 +774,63 @@ if __name__ == "__main__":
             print(f"\n🔍 Query {i+1}/{len(queries)} in {category} category:")
             print(f"   Text: {query_text}")
             
-            # Test single query
-            result = evaluator.measure_single_query_latency(query_text, category)
-            
-            # Add to accumulator for chart generation
-            evaluator.add_result_to_accumulator(result)
-            
-            # Save individual result
-            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-            filename = f"latency_{category}_query{i+1}_{timestamp}.json"
-            saved_path = evaluator.save_single_result(result, filename)
-            
-            # Show summary
-            if result.get('success'):
-                print(f"   ✅ Success: {result['total_latency']:.2f}s total")
-                print(f"      Breakdown: Extract={result['timing_details']['step1_condition_extraction']:.2f}s, "
-                      f"Retrieve={result['timing_details']['step3_retrieval']:.2f}s, "
-                      f"Generate={result['timing_details']['step4_generation']:.2f}s")
-            else:
-                print(f"   ❌ Failed: {result.get('status')} - {result.get('error', 'Unknown error')}")
+            # Comprehensive evaluation (collects all metrics 1-4)
+            result = evaluator.evaluate_single_query_comprehensive(query_text, category)
             
             # Pause between queries to avoid rate limits
-            if i < len(queries) - 1:  # Not the last query in category
+            if i < len(queries) - 1:
                 print(f"   ⏳ Pausing 5s before next query...")
                 time.sleep(5)
         
         # Longer pause between categories
-        if category != list(queries_by_category.keys())[-1]:  # Not the last category
+        if category != list(queries_by_category.keys())[-1]:
             print(f"\n⏳ Pausing 10s before next category...")
             time.sleep(10)
     
-    # Generate comprehensive analysis (no charts - pure data)
-    print(f"\n📊 Generating comprehensive statistical summary...")
-    
-    # Calculate category statistics
-    final_stats = evaluator.calculate_category_statistics()
+    # Generate and save all metric statistics
+    print(f"\n📊 Generating comprehensive analysis for all metrics...")
     
-    # Save statistics for chart generation
-    stats_path = evaluator.save_statistics_summary()
+    # Save separate statistics for each metric
+    saved_stats = evaluator.save_all_metric_statistics()
     
     # Save medical outputs for model comparison
     outputs_path = evaluator.save_medical_outputs()
     
-    # Print final summary
-    print(f"\n📊 === FINAL LATENCY ANALYSIS SUMMARY ===")
-    category_results = final_stats['category_results']
-    overall_results = final_stats['overall_results']
-    
-    print(f"Overall Performance:")
-    print(f"   Average Latency: {overall_results['average_latency']:.2f}s (±{overall_results['std_deviation']:.2f})")
-    print(f"   Success Rate: {overall_results['successful_queries']}/{overall_results['total_queries']}")
-    print(f"   30s Target Compliance: {overall_results['target_compliance']:.1%}")
-    
-    print(f"\nCategory Breakdown:")
-    for category, stats in category_results.items():
-        if stats['query_count'] > 0:
-            print(f"   {category.capitalize()}: {stats['average_latency']:.2f}s (±{stats['std_deviation']:.2f}) [{stats['query_count']} queries]")
-    
-    print(f"\n✅ Data collection complete! Files saved:")
-    print(f"   📊 Statistics: {stats_path}")
-    print(f"   📝 Medical Outputs: {outputs_path}")
-    print(f"   📁 Individual results: {Path(__file__).parent / 'results'}")
-    print(f"\n💡 Next step: Run latency_chart_generator.py to create visualizations")
-    
-    def add_result_to_accumulator(self, result: Dict[str, Any]):
-        """Add successful result to category accumulator"""
-        if result.get('success') and result.get('category') in self.accumulated_results:
-            category = result['category']
-            self.accumulated_results[category].append(result)
-            print(f"📊 Added result to {category} category. Total: {len(self.accumulated_results[category])}")
+    # Save comprehensive details
+    details_path = evaluator.save_comprehensive_details()
     
-    def save_statistics_summary(self, filename: str = None) -> str:
-        """Save statistical summary for chart generation"""
-        stats = self.calculate_category_statistics()
-        
-        if filename is None:
-            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-            filename = f"latency_statistics_{timestamp}.json"
-        
-        # Ensure results directory exists
-        results_dir = Path(__file__).parent / "results"
-        results_dir.mkdir(exist_ok=True)
-        
-        filepath = results_dir / filename
-        
-        with open(filepath, 'w', encoding='utf-8') as f:
-            json.dump(stats, f, indent=2, ensure_ascii=False)
-        
-        print(f"📊 Statistics saved to: {filepath}")
-        return str(filepath)
+    # Print comprehensive summary
+    print(f"\n📊 === COMPREHENSIVE EVALUATION SUMMARY ===")
     
-    def save_medical_outputs(self, filename: str = None) -> str:
-        """Save medical advice outputs for model comparison"""
-        if filename is None:
-            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-            filename = f"medical_outputs_{timestamp}.json"
+    for metric_name in ["latency", "extraction", "relevance", "coverage"]:
+        stats = evaluator.calculate_metric_statistics(metric_name)
+        overall_results = stats['overall_results']
         
-        # Ensure results directory exists
-        results_dir = Path(__file__).parent / "results"
-        results_dir.mkdir(exist_ok=True)
+        print(f"\n{metric_name.upper()} METRICS:")
         
-        filepath = results_dir / filename
+        if metric_name == "latency":
+            print(f"   Average: {overall_results['average_latency']:.2f}s (±{overall_results['std_deviation']:.2f})")
+            print(f"   30s Target: {'✅ Met' if overall_results['target_compliance'] >= 0.8 else '❌ Not Met'}")
         
-        # Create comprehensive output data
-        output_data = {
-            "evaluation_metadata": {
-                "total_outputs": len(self.medical_outputs),
-                "categories": list(set(output['category'] for output in self.medical_outputs)),
-                "timestamp": datetime.now().isoformat(),
-                "model_type": "Med42-70B_RAG_enhanced"  # For future comparison
-            },
-            "medical_outputs": self.medical_outputs
-        }
+        elif metric_name == "extraction":
+            print(f"   Success Rate: {overall_results['success_rate']:.1%}")
+            print(f"   80% Target: {'✅ Met' if overall_results['target_compliance'] else '❌ Not Met'}")
         
-        with open(filepath, 'w', encoding='utf-8') as f:
-            json.dump(output_data, f, indent=2, ensure_ascii=False)
-        
-        print(f"📝 Medical outputs saved to: {filepath}")
-        print(f"    Total outputs: {len(self.medical_outputs)}")
-        print(f"    Categories: {', '.join(set(output['category'] for output in self.medical_outputs))}")
+        elif metric_name == "relevance":
+            print(f"   Average Relevance: {overall_results['average_relevance']:.3f}")
+            print(f"   0.25 Target: {'✅ Met' if overall_results.get('target_compliance', False) else '❌ Not Met'}")
         
-        return str(filepath)
+        elif metric_name == "coverage":
+            print(f"   Average Coverage: {overall_results['average_coverage']:.3f} ({overall_results['average_coverage']*100:.1f}%)")
+            print(f"   60% Target: {'✅ Met' if overall_results['meets_threshold'] else '❌ Not Met'}")
     
-    def calculate_category_statistics(self) -> Dict[str, Any]:
-        """Calculate statistics for each category and overall"""
-        category_stats = {}
-        all_successful_latencies = []
-        
-        for category, results in self.accumulated_results.items():
-            latencies = [r['total_latency'] for r in results if r.get('success')]
-            
-            if latencies:
-                category_stats[category] = {
-                    "average_latency": sum(latencies) / len(latencies),
-                    "std_deviation": self._calculate_std(latencies),
-                    "min_latency": min(latencies),
-                    "max_latency": max(latencies),
-                    "query_count": len(latencies),
-                    "individual_latencies": latencies
-                }
-                all_successful_latencies.extend(latencies)
-            else:
-                category_stats[category] = {
-                    "average_latency": 0.0,
-                    "std_deviation": 0.0,
-                    "min_latency": 0.0,
-                    "max_latency": 0.0,
-                    "query_count": 0,
-                    "individual_latencies": []
-                }
-        
-        # Calculate overall statistics
-        overall_stats = {
-            "average_latency": sum(all_successful_latencies) / len(all_successful_latencies) if all_successful_latencies else 0.0,
-            "std_deviation": self._calculate_std(all_successful_latencies),
-            "min_latency": min(all_successful_latencies) if all_successful_latencies else 0.0,
-            "max_latency": max(all_successful_latencies) if all_successful_latencies else 0.0,
-            "total_queries": sum(len(results) for results in self.accumulated_results.values()),
-            "successful_queries": len(all_successful_latencies),
-            "target_compliance": sum(1 for lat in all_successful_latencies if lat <= 30.0) / len(all_successful_latencies) if all_successful_latencies else 0.0
-        }
-        
-        return {
-            "category_results": category_stats,
-            "overall_results": overall_stats,
-            "timestamp": datetime.now().isoformat()
-        }
-    
-    def _calculate_std(self, values: List[float]) -> float:
-        """Calculate standard deviation"""
-        if len(values) < 2:
-            return 0.0
-        
-        mean = sum(values) / len(values)
-        variance = sum((x - mean) ** 2 for x in values) / len(values)
-        return variance ** 0.5
+    print(f"\n✅ Comprehensive evaluation complete! Files saved:")
+    for metric_name, filepath in saved_stats.items():
+        print(f"   📊 {metric_name.capitalize()}: {filepath}")
+    print(f"   📝 Medical Outputs: {outputs_path}")
+    print(f"   📋 Comprehensive Details: {details_path}")
+    print(f"\n💡 Next step: Run chart generators for individual metrics")
+    print(f"   python latency_chart_generator.py")
+    print(f"   python extraction_chart_generator.py  # (create separately)")
+    print(f"   python relevance_chart_generator.py   # (create separately)")
+    print(f"   python coverage_chart_generator.py    # (create separately)")