Merge pull request #12 from YanBoChen0928/Merged20250805

evaluation/direct_llm_evaluator.py

@@ -0,0 +1,419 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Direct LLM Evaluator (Med42-70B Only)
+========================================================
+
+Tests Med42-70B directly without RAG pipeline.
+Only applicable metrics: 1 (Latency), 5 (Actionability), 6 (Evidence Quality)
+
+Metrics 2-4 (Extraction, Relevance, Coverage) are not applicable for direct LLM.
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import time
+import json
+import os
+import sys
+from typing import Dict, List, Any
+from datetime import datetime
+from pathlib import Path
+import re
+
+# Add project path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import LLM client only (no retrieval system needed)
+try:
+    from llm_clients import llm_Med42_70BClient
+except ImportError as e:
+    print(f"❌ Import failed: {e}")
+    print("Please ensure running from project root directory")
+    sys.exit(1)
+
+
+class DirectLLMEvaluator:
+    """Direct LLM evaluation without RAG pipeline"""
+    
+    def __init__(self):
+        """Initialize direct LLM client only"""
+        print("🔧 Initializing Direct LLM Evaluator...")
+        
+        # Initialize only LLM client (no retrieval, no user_prompt processing)
+        self.llm_client = llm_Med42_70BClient()
+        
+        # Results accumulation
+        self.direct_results = []
+        self.medical_outputs = []
+        
+        print("✅ Direct LLM Evaluator initialization complete")
+    
+    def evaluate_direct_llm_query(self, query: str, category: str = "unknown") -> Dict[str, Any]:
+        """
+        Direct LLM evaluation for single query
+        
+        Only tests direct LLM response without RAG pipeline
+        Applicable metrics: 1 (Latency), 5-6 (via medical output)
+        
+        Args:
+            query: Medical query to test
+            category: Query category (diagnosis/treatment/mixed)
+        """
+        print(f"🔍 Direct LLM evaluation: {query[:50]}...")
+        print(f"📋 Category: {category}")
+        
+        overall_start = time.time()
+        
+        try:
+            # Direct LLM call without any RAG processing
+            llm_start = time.time()
+            
+            # Create direct medical consultation prompt
+            direct_prompt = f"""
+You are a medical expert providing clinical guidance.
+
+Patient Query: {query}
+
+Please provide comprehensive medical advice including:
+1. Differential diagnosis (if applicable)
+2. Immediate assessment steps
+3. Treatment recommendations
+4. Clinical considerations
+
+Provide evidence-based, actionable medical guidance.
+"""
+            
+            # Direct LLM generation (same parameters as RAG system for fair comparison)
+            response = self.llm_client.analyze_medical_query(
+                query=direct_prompt,
+                max_tokens=1600,  # Same as RAG system primary setting  
+                timeout=60.0      # Increased timeout for stable evaluation
+            )
+            # Extract medical advice from response (Med42 client returns dict with 'raw_response')
+            if isinstance(response, dict):
+                medical_advice = response.get('raw_response', '') or response.get('content', '')
+            else:
+                medical_advice = str(response)
+            
+            llm_time = time.time() - llm_start
+            total_time = time.time() - overall_start
+            
+            # Check if response is valid (not empty) - focus on content, not timeout
+            if not medical_advice or len(medical_advice.strip()) == 0:
+                print(f"❌ Direct LLM returned empty response after {total_time:.2f}s")
+                raise ValueError("Empty response from LLM - no content generated")
+            
+            # Create result
+            result = {
+                "query": query,
+                "category": category,
+                
+                # Metric 1: Total Latency (direct LLM call time)
+                "latency_metrics": {
+                    "total_latency": total_time,
+                    "llm_generation_time": llm_time,
+                    "meets_target": total_time <= 60.0
+                },
+                
+                # Metrics 2-4: Not applicable for direct LLM
+                "extraction_metrics": {
+                    "not_applicable": True,
+                    "reason": "No extraction pipeline in direct LLM"
+                },
+                "relevance_metrics": {
+                    "not_applicable": True,
+                    "reason": "No retrieval pipeline in direct LLM"
+                },
+                "coverage_metrics": {
+                    "not_applicable": True,
+                    "reason": "No retrieval content to cover"
+                },
+                
+                # Medical advice for metrics 5-6 evaluation
+                "medical_advice": medical_advice,
+                "advice_length": len(medical_advice),
+                
+                "overall_success": True,
+                "model_type": "Med42-70B_direct",
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            # Store result
+            self.direct_results.append(result)
+            
+            # Store medical output for LLM judge evaluation
+            medical_output = {
+                "query": query,
+                "category": category,
+                "medical_advice": medical_advice,
+                "query_id": f"{category}_query_direct",
+                "model_type": "Med42-70B_direct",
+                "processing_time": total_time,
+                "timestamp": datetime.now().isoformat()
+            }
+            self.medical_outputs.append(medical_output)
+            
+            print(f"✅ Direct LLM completed in {total_time:.2f}s")
+            print(f"📝 Generated advice: {len(medical_advice)} characters")
+            
+            return result
+            
+        except Exception as e:
+            total_time = time.time() - overall_start
+            print(f"❌ Direct LLM evaluation failed after {total_time:.2f}s: {e}")
+            
+            error_result = {
+                "query": query,
+                "category": category,
+                "latency_metrics": {
+                    "total_latency": total_time,
+                    "meets_target": False
+                },
+                "overall_success": False,
+                "error": str(e),
+                "model_type": "Med42-70B_direct",
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            self.direct_results.append(error_result)
+            
+            # Do NOT add failed queries to medical_outputs for judge evaluation
+            # Only successful queries with valid medical advice should be evaluated
+            
+            return error_result
+    
+    def parse_queries_from_file(self, filepath: str) -> Dict[str, List[Dict]]:
+        """Parse queries from file with category labels"""
+        print(f"📁 Reading queries from file: {filepath}")
+        
+        try:
+            with open(filepath, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            queries_by_category = {
+                "diagnosis": [],
+                "treatment": [], 
+                "mixed": []
+            }
+            
+            lines = content.strip().split('\n')
+            
+            for line in lines:
+                line = line.strip()
+                if not line:
+                    continue
+                
+                match = re.match(r'^\d+\.(diagnosis|treatment|mixed/complicated|mixed):\s*(.+)', line, re.IGNORECASE)
+                if match:
+                    category_raw = match.group(1).lower()
+                    query_text = match.group(2).strip()
+                    
+                    if category_raw in ['mixed/complicated', 'mixed']:
+                        category = 'mixed'
+                    else:
+                        category = category_raw
+                    
+                    if category in queries_by_category and len(query_text) > 15:
+                        queries_by_category[category].append({
+                            "text": query_text,
+                            "category": category
+                        })
+            
+            print(f"📋 Parsed queries by category:")
+            for category, category_queries in queries_by_category.items():
+                print(f"  {category.capitalize()}: {len(category_queries)} queries")
+            
+            return queries_by_category
+            
+        except Exception as e:
+            print(f"❌ Failed to read file: {e}")
+            return {"error": f"Failed to read file: {e}"}
+    
+    def calculate_direct_llm_statistics(self) -> Dict[str, Any]:
+        """Calculate statistics for direct LLM evaluation"""
+        successful_results = [r for r in self.direct_results if r.get('overall_success')]
+        
+        if successful_results:
+            latencies = [r['latency_metrics']['total_latency'] for r in successful_results]
+            
+            # Category-wise statistics
+            category_stats = {}
+            results_by_category = {"diagnosis": [], "treatment": [], "mixed": []}
+            
+            for result in successful_results:
+                category = result.get('category', 'unknown')
+                if category in results_by_category:
+                    results_by_category[category].append(result)
+            
+            for category, results in results_by_category.items():
+                if results:
+                    cat_latencies = [r['latency_metrics']['total_latency'] for r in results]
+                    category_stats[category] = {
+                        "average_latency": sum(cat_latencies) / len(cat_latencies),
+                        "query_count": len(cat_latencies),
+                        "target_compliance": sum(1 for lat in cat_latencies if lat <= 60.0) / len(cat_latencies)
+                    }
+                else:
+                    category_stats[category] = {
+                        "average_latency": 0.0,
+                        "query_count": 0,
+                        "target_compliance": 0.0
+                    }
+            
+            # Overall statistics
+            overall_stats = {
+                "average_latency": sum(latencies) / len(latencies),
+                "min_latency": min(latencies),
+                "max_latency": max(latencies),
+                "successful_queries": len(successful_results),
+                "total_queries": len(self.direct_results),
+                "success_rate": len(successful_results) / len(self.direct_results),
+                "target_compliance": sum(1 for lat in latencies if lat <= 60.0) / len(latencies)
+            }
+        else:
+            category_stats = {cat: {"average_latency": 0.0, "query_count": 0, "target_compliance": 0.0} 
+                            for cat in ["diagnosis", "treatment", "mixed"]}
+            overall_stats = {
+                "average_latency": 0.0,
+                "successful_queries": 0,
+                "total_queries": len(self.direct_results),
+                "success_rate": 0.0,
+                "target_compliance": 0.0
+            }
+        
+        return {
+            "category_results": category_stats,
+            "overall_results": overall_stats,
+            "model_type": "Med42-70B_direct",
+            "timestamp": datetime.now().isoformat()
+        }
+    
+    def save_direct_llm_statistics(self, filename: str = None) -> str:
+        """Save direct LLM statistics"""
+        stats = self.calculate_direct_llm_statistics()
+        
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"direct_llm_statistics_{timestamp}.json"
+        
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        filepath = results_dir / filename
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(stats, f, indent=2, ensure_ascii=False)
+        
+        print(f"📊 Direct LLM statistics saved to: {filepath}")
+        return str(filepath)
+    
+    def save_direct_medical_outputs(self, filename: str = None) -> str:
+        """Save medical outputs for LLM judge evaluation"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"medical_outputs_direct_{timestamp}.json"
+        
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        filepath = results_dir / filename
+        
+        output_data = {
+            "evaluation_metadata": {
+                "total_outputs": len(self.medical_outputs),
+                "categories": list(set(output['category'] for output in self.medical_outputs)),
+                "timestamp": datetime.now().isoformat(),
+                "model_type": "Med42-70B_direct"
+            },
+            "medical_outputs": self.medical_outputs
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(output_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"📝 Direct medical outputs saved to: {filepath}")
+        return str(filepath)
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent direct LLM evaluation interface"""
+    
+    print("🚀 OnCall.ai Direct LLM Evaluator - Med42-70B Only")
+    
+    if len(sys.argv) > 1:
+        query_file = sys.argv[1]
+    else:
+        # Default to evaluation/single_test_query.txt for consistency
+        query_file = Path(__file__).parent / "single_test_query.txt"
+    
+    if not os.path.exists(query_file):
+        print(f"❌ Query file not found: {query_file}")
+        print("Usage: python direct_llm_evaluator.py [query_file.txt]")
+        sys.exit(1)
+    
+    # Initialize evaluator
+    evaluator = DirectLLMEvaluator()
+    
+    # Parse queries
+    queries_by_category = evaluator.parse_queries_from_file(str(query_file))
+    
+    if "error" in queries_by_category:
+        print(f"❌ Failed to parse queries: {queries_by_category['error']}")
+        sys.exit(1)
+    
+    # Test direct LLM for each query
+    print(f"\n🧪 Direct LLM Testing (No RAG Pipeline)")
+    
+    for category, queries in queries_by_category.items():
+        if not queries:
+            continue
+            
+        print(f"\n📂 Testing {category.upper()} with direct Med42-70B:")
+        
+        for i, query_info in enumerate(queries):
+            query_text = query_info['text']
+            
+            # Direct LLM evaluation
+            result = evaluator.evaluate_direct_llm_query(query_text, category)
+            
+            # Pause between queries
+            if i < len(queries) - 1:
+                print(f"   ⏳ Pausing 5s before next query...")
+                time.sleep(5)
+        
+        # Pause between categories
+        if category != list(queries_by_category.keys())[-1]:
+            print(f"\n⏳ Pausing 10s before next category...")
+            time.sleep(10)
+    
+    # Save results
+    print(f"\n📊 Generating direct LLM analysis...")
+    
+    stats_path = evaluator.save_direct_llm_statistics()
+    outputs_path = evaluator.save_direct_medical_outputs()
+    
+    # Print summary
+    stats = evaluator.calculate_direct_llm_statistics()
+    overall_results = stats['overall_results']
+    
+    print(f"\n📊 === DIRECT LLM EVALUATION SUMMARY ===")
+    print(f"Overall Performance:")
+    print(f"   Average Latency: {overall_results['average_latency']:.2f}s")
+    print(f"   Success Rate: {overall_results['successful_queries']}/{overall_results['total_queries']}")
+    print(f"   60s Target Compliance: {overall_results['target_compliance']:.1%}")
+    
+    print(f"\nApplicable Metrics:")
+    print(f"   ✅ Metric 1 (Latency): Measured")
+    print(f"   ❌ Metric 2 (Extraction): Not applicable - no extraction pipeline")
+    print(f"   ❌ Metric 3 (Relevance): Not applicable - no retrieval pipeline")
+    print(f"   ❌ Metric 4 (Coverage): Not applicable - no retrieval content")
+    print(f"   🔄 Metric 5 (Actionability): Requires LLM judge evaluation")
+    print(f"   🔄 Metric 6 (Evidence): Requires LLM judge evaluation")
+    
+    print(f"\n✅ Direct LLM evaluation complete!")
+    print(f"📊 Statistics: {stats_path}")
+    print(f"📝 Medical Outputs: {outputs_path}")
+    print(f"\n💡 Next step: Run python metric5_6_llm_judge_evaluator.py rag,direct for metrics 5-6")

evaluation/latency_evaluator.py

@@ -0,0 +1,892 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Comprehensive Evaluator (Metrics 1-8)
+========================================================
+
+Single execution to collect all metrics 1-4 data from app.py pipeline.
+Generates foundation data for metrics 5-8 evaluation in downstream processors.
+
+COMPLETE METRICS OVERVIEW:
+
+PIPELINE PERFORMANCE METRICS (Collected by this evaluator):
+1. Total Latency (總處理時長) - Complete pipeline processing time from query to response
+2. Condition Extraction Success Rate (條件抽取成功率) - Success rate of user_prompt.py condition extraction
+3. Retrieval Relevance (檢索相關性) - Average cosine similarity scores from retrieval.py results
+4. Retrieval Coverage (檢索覆蓋率) - Medical keyword utilization rate between retrieved content and generated advice
+
+LLM JUDGE METRICS (Processed by metric5_6_llm_judge_evaluator.py):
+5. Clinical Actionability (臨床可操作性) - Third-party LLM evaluation of medical advice actionability (1-10 scale)
+   * Uses batch evaluation strategy with Llama3-70B as judge
+   * Measures: Can healthcare providers immediately act on this advice?
+   * Target threshold: ≥7.0/10 for acceptable actionability
+   
+6. Clinical Evidence Quality (臨床證據品質) - Third-party LLM evaluation of evidence-based quality (1-10 scale)
+   * Uses same batch evaluation call as metric 5 for efficiency
+   * Measures: Is the advice evidence-based and follows medical standards?
+   * Target threshold: ≥7.5/10 for acceptable evidence quality
+
+RETRIEVAL PRECISION METRICS (Processed by metric7_8_precision_MRR.py):
+7. Precision@K (檢索精確率) - Proportion of relevant results in top-K retrieval results
+   * Uses adaptive threshold based on query complexity (0.15 for complex, 0.25 for simple queries)
+   * Query complexity determined by unique emergency keywords count (≥4 = complex)
+   * Measures: relevant_results / total_retrieved_results
+   
+8. Mean Reciprocal Rank (平均倒數排名) - Average reciprocal rank of first relevant result
+   * Uses same adaptive threshold as Precision@K
+   * Measures: 1 / rank_of_first_relevant_result (0 if no relevant results)
+   * Higher MRR indicates relevant results appear earlier in ranking
+
+DATA FLOW ARCHITECTURE:
+1. latency_evaluator.py → comprehensive_details_*.json (metrics 1-4 + pipeline data)
+2. latency_evaluator.py → medical_outputs_*.json (medical advice for judge evaluation)
+3. metric5_6_llm_judge_evaluator.py → judge_evaluation_*.json (metrics 5-6)
+4. metric7_8_precision_MRR.py → precision_mrr_analysis_*.json (metrics 7-8)
+
+Note: This evaluator focuses on metrics 1-4 collection. Metrics 5-8 require separate downstream evaluation.
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import time
+import json
+import os
+import sys
+from typing import Dict, List, Any, Set
+from datetime import datetime
+from pathlib import Path
+import re
+
+# Add project path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import existing system components
+try:
+    from user_prompt import UserPromptProcessor
+    from retrieval import BasicRetrievalSystem
+    from llm_clients import llm_Med42_70BClient
+    from generation import MedicalAdviceGenerator
+except ImportError as e:
+    print(f"❌ Import failed: {e}")
+    print("Please ensure running from project root directory")
+    sys.exit(1)
+
+
+class ComprehensiveEvaluator:
+    """Comprehensive evaluator for metrics 1-4 - single execution approach"""
+    
+    def __init__(self):
+        """Initialize system components (identical to app.py)"""
+        print("🔧 Initializing Comprehensive Evaluator...")
+        
+        # Initialize existing system components (same as app.py)
+        self.llm_client = llm_Med42_70BClient()
+        self.retrieval_system = BasicRetrievalSystem()
+        self.user_prompt_processor = UserPromptProcessor(
+            llm_client=self.llm_client,
+            retrieval_system=self.retrieval_system
+        )
+        self.medical_generator = MedicalAdviceGenerator(llm_client=self.llm_client)
+        
+        # Results accumulation for all metrics
+        self.comprehensive_results = []
+        self.medical_outputs = []
+        
+        print("✅ Comprehensive Evaluator initialization complete")
+    
+    def extract_medical_keywords(self, text: str) -> Set[str]:
+        """Extract medical keywords for coverage analysis"""
+        if not text:
+            return set()
+        
+        medical_keywords = set()
+        text_lower = text.lower()
+        
+        # Medical terminology patterns
+        patterns = [
+            r'\b[a-z]+(?:osis|itis|pathy|emia|uria|gram|scopy)\b',  # Medical suffixes
+            r'\b(?:cardio|neuro|pulmo|gastro|hepato|nephro)[a-z]+\b',  # Medical prefixes
+            r'\b(?:diagnosis|treatment|therapy|intervention|management)\b',  # Medical actions
+            r'\b(?:patient|symptom|condition|disease|disorder|syndrome)\b',  # Medical entities
+            r'\b(?:acute|chronic|severe|mild|moderate|emergency)\b',  # Medical descriptors
+            r'\b[a-z]+(?:al|ic|ous|ive)\s+(?:pain|failure|infection|injury)\b',  # Compound terms
+            r'\b(?:ecg|ekg|ct|mri|x-ray|ultrasound|biopsy)\b',  # Medical procedures
+            r'\b\d+\s*(?:mg|ml|units|hours|days|minutes)\b',  # Dosages and timeframes
+        ]
+        
+        for pattern in patterns:
+            matches = re.findall(pattern, text_lower)
+            medical_keywords.update(match.strip() for match in matches)
+        
+        # Additional common medical terms
+        common_medical_terms = [
+            'blood', 'pressure', 'heart', 'chest', 'pain', 'stroke', 'seizure',
+            'emergency', 'hospital', 'monitor', 'assess', 'evaluate', 'immediate',
+            'protocol', 'guideline', 'recommendation', 'risk', 'factor'
+        ]
+        
+        for term in common_medical_terms:
+            if term in text_lower:
+                medical_keywords.add(term)
+        
+        # Filter out very short terms and common words
+        filtered_keywords = {
+            kw for kw in medical_keywords 
+            if len(kw) > 2 and kw not in ['the', 'and', 'for', 'with', 'are', 'can', 'may']
+        }
+        
+        return filtered_keywords
+    
+    def calculate_coverage_metrics(self, generated_advice: str, retrieval_results: List[Dict]) -> Dict[str, Any]:
+        """Calculate coverage metrics from generated advice and retrieval results"""
+        if not generated_advice or not retrieval_results:
+            return {
+                "coverage_score": 0.0,
+                "matched_keywords": [],
+                "advice_keywords": [],
+                "source_keywords": [],
+                "coverage_percentage": 0.0,
+                "meets_threshold": False
+            }
+        
+        # Extract keywords from generated advice
+        advice_keywords = self.extract_medical_keywords(generated_advice)
+        
+        # Extract keywords from all retrieved documents
+        all_source_keywords = set()
+        for doc in retrieval_results:
+            doc_content = doc.get('content', '') or doc.get('text', '')
+            doc_keywords = self.extract_medical_keywords(doc_content)
+            all_source_keywords.update(doc_keywords)
+        
+        # Calculate coverage
+        matched_keywords = advice_keywords.intersection(all_source_keywords)
+        coverage_score = len(matched_keywords) / len(all_source_keywords) if all_source_keywords else 0.0
+        
+        return {
+            "coverage_score": coverage_score,
+            "matched_keywords": list(matched_keywords),
+            "advice_keywords": list(advice_keywords),
+            "source_keywords": list(all_source_keywords),
+            "advice_keywords_count": len(advice_keywords),
+            "source_keywords_count": len(all_source_keywords),
+            "matched_keywords_count": len(matched_keywords),
+            "coverage_percentage": coverage_score * 100,
+            "meets_threshold": coverage_score >= 0.4
+        }
+    
+    def evaluate_single_query_comprehensive(self, query: str, category: str = "unknown") -> Dict[str, Any]:
+        """
+        Comprehensive evaluation for single query - collects all metrics 1-4 data
+        
+        Replicates app.py's process_medical_query pipeline exactly
+        
+        Args:
+            query: Medical query to test
+            category: Query category (diagnosis/treatment/mixed)
+        """
+        print(f"🔍 Comprehensive evaluation: {query[:50]}...")
+        print(f"📋 Category: {category}")
+        
+        overall_start = time.time()
+        timing_details = {}
+        
+        try:
+            # STEP 1: Query Processing and Condition Extraction (identical to app.py)
+            step1_start = time.time()
+            condition_result = self.user_prompt_processor.extract_condition_keywords(query)
+            step1_time = time.time() - step1_start
+            timing_details['step1_condition_extraction'] = step1_time
+            
+            print(f"   Step 1 - Condition extraction: {step1_time:.3f}s")
+            print(f"   Extracted condition: {condition_result.get('condition', 'None')}")
+            
+            # Check if valid medical query
+            if condition_result.get('query_status') in ['invalid_query', 'non_medical']:
+                total_time = time.time() - overall_start
+                return self._create_failed_result(query, category, total_time, timing_details, 
+                                                "non_medical", condition_result)
+            
+            # STEP 2: User Confirmation (simulate auto-confirmation)
+            step2_start = time.time()
+            confirmation = self.user_prompt_processor.handle_user_confirmation(condition_result)
+            step2_time = time.time() - step2_start
+            timing_details['step2_confirmation'] = step2_time
+            
+            if not condition_result.get('condition'):
+                total_time = time.time() - overall_start
+                return self._create_failed_result(query, category, total_time, timing_details,
+                                                "no_condition", condition_result)
+            
+            # STEP 3: Medical Guidelines Retrieval (identical to app.py)
+            step3_start = time.time()
+            
+            search_query = f"{condition_result.get('emergency_keywords', '')} {condition_result.get('treatment_keywords', '')}".strip()
+            if not search_query:
+                search_query = condition_result.get('condition', query)
+            
+            retrieval_results = self.retrieval_system.search(search_query, top_k=5)
+            step3_time = time.time() - step3_start
+            timing_details['step3_retrieval'] = step3_time
+            
+            processed_results = retrieval_results.get('processed_results', [])
+            print(f"   Step 3 - Retrieval: {step3_time:.3f}s ({len(processed_results)} results)")
+            
+            # STEP 4: Medical Advice Generation (identical to app.py)
+            step4_start = time.time()
+            
+            intention = self._detect_query_intention(query)
+            medical_advice_result = self.medical_generator.generate_medical_advice(
+                user_query=query,
+                retrieval_results=retrieval_results,
+                intention=intention
+            )
+            step4_time = time.time() - step4_start
+            timing_details['step4_generation'] = step4_time
+            
+            generated_advice = medical_advice_result.get('medical_advice', '')
+            confidence_score = medical_advice_result.get('confidence_score', 0.0)
+            
+            print(f"   Step 4 - Generation: {step4_time:.3f}s")
+            
+            total_time = time.time() - overall_start
+            
+            # METRIC 2: Condition Extraction Analysis
+            extraction_success = (
+                condition_result.get('condition') and
+                condition_result.get('condition') != "unknown" and
+                condition_result.get('query_status') not in ['invalid_query', 'non_medical']
+            )
+            
+            extraction_metrics = {
+                "extraction_success": extraction_success,
+                "extracted_condition": condition_result.get('condition'),
+                "query_status": condition_result.get('query_status'),
+                "emergency_keywords": condition_result.get('emergency_keywords', []),
+                "treatment_keywords": condition_result.get('treatment_keywords', []),
+                "fallback_level": condition_result.get('fallback_level', 'unknown'),
+                "extraction_time": step1_time
+            }
+            
+            # METRIC 3: Retrieval Relevance Analysis
+            if processed_results:
+                relevance_scores = []
+                for doc_result in processed_results:
+                    # Get angular distance and convert to relevance using correct formula
+                    distance = doc_result.get('distance', 1.0)
+                    relevance = 1.0 - (distance**2) / 2.0  # Correct mathematical conversion
+                    relevance_scores.append(relevance)
+                
+                average_relevance = sum(relevance_scores) / len(relevance_scores)
+                high_relevance_count = sum(1 for score in relevance_scores if score >= 0.85)
+                
+                relevance_metrics = {
+                    "average_relevance": average_relevance,
+                    "max_relevance": max(relevance_scores),
+                    "min_relevance": min(relevance_scores),
+                    "relevance_scores": relevance_scores,
+                    "high_relevance_count": high_relevance_count,
+                    "high_relevance_ratio": high_relevance_count / len(relevance_scores),
+                    "retrieved_count": len(processed_results),
+                    "meets_threshold": average_relevance >= 0.85,
+                    "retrieval_time": step3_time
+                }
+            else:
+                relevance_metrics = {
+                    "average_relevance": 0.0,
+                    "max_relevance": 0.0,
+                    "min_relevance": 0.0,
+                    "similarity_scores": [],
+                    "high_relevance_count": 0,
+                    "high_relevance_ratio": 0.0,
+                    "retrieved_count": 0,
+                    "meets_threshold": False,
+                    "retrieval_time": step3_time
+                }
+            
+            # METRIC 4: Retrieval Coverage Analysis
+            coverage_metrics = self.calculate_coverage_metrics(generated_advice, processed_results)
+            coverage_metrics["generation_time"] = step4_time
+            
+            # Create comprehensive result
+            comprehensive_result = {
+                "query": query,
+                "category": category,
+                
+                # Metric 1: Total Latency - Complete pipeline processing time
+                "latency_metrics": {
+                    "total_latency": total_time,
+                    "timing_details": timing_details,
+                    "meets_target": total_time <= 60.0
+                },
+                
+                # Metric 2: Condition Extraction - Success rate from user_prompt.py
+                "extraction_metrics": extraction_metrics,
+                
+                # Metric 3: Retrieval Relevance - Cosine similarity from retrieval.py  
+                "relevance_metrics": relevance_metrics,
+                
+                # Metric 4: Retrieval Coverage - Advice utilization of retrieved content
+                "coverage_metrics": coverage_metrics,
+                
+                # Complete pipeline data (for debugging and detailed analysis)
+                "pipeline_data": {
+                    "condition_result": condition_result,
+                    "retrieval_results": retrieval_results,
+                    "medical_advice_result": medical_advice_result,
+                    "search_query": search_query,
+                    "intention": intention
+                },
+                
+                "overall_success": True,
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            # Validate data completeness for metrics 7-8 analysis
+            ready = True
+            data = comprehensive_result.get('pipeline_data', {})
+            
+            # 1. Check retrieval results completeness for precision/MRR calculation
+            retr = data.get('retrieval_results', {}).get('processed_results', [])
+            if not retr or 'distance' not in retr[0]:
+                ready = False
+            
+            # 2. Check condition extraction completeness for complexity analysis
+            cond = data.get('condition_result', {}).get('condition')
+            if not cond:
+                ready = False
+            
+            # 3. Check overall execution status
+            if not comprehensive_result.get('overall_success', False):
+                ready = False
+            
+            # 4. Check retrieval timing data completeness
+            if 'retrieval_time' not in comprehensive_result.get('relevance_metrics', {}):
+                ready = False
+            
+            # Set metrics 7-8 readiness flag for downstream precision/MRR analysis
+            comprehensive_result['precision_mrr_ready'] = ready
+            
+            # Store result
+            self.comprehensive_results.append(comprehensive_result)
+            
+            # Store medical output for model comparison
+            medical_output = {
+                "query": query,
+                "category": category,
+                "medical_advice": generated_advice,
+                "confidence_score": confidence_score,
+                "query_id": f"{category}_query",
+                "processing_time": total_time,
+                "timestamp": datetime.now().isoformat()
+            }
+            self.medical_outputs.append(medical_output)
+            
+            print(f"✅ Comprehensive evaluation completed in {total_time:.2f}s")
+            print(f"   📊 Metrics: Latency={total_time:.2f}s, Extraction={'✅' if extraction_success else '❌'}, "
+                  f"Relevance={average_relevance:.3f}, Coverage={coverage_metrics['coverage_score']:.3f}")
+            
+            return comprehensive_result
+            
+        except Exception as e:
+            total_time = time.time() - overall_start
+            print(f"❌ Comprehensive evaluation failed after {total_time:.2f}s: {e}")
+            
+            return self._create_failed_result(query, category, total_time, timing_details, "error", None, str(e))
+    
+    def _create_failed_result(self, query: str, category: str, total_time: float, 
+                            timing_details: Dict, status: str, condition_result: Dict = None, 
+                            error: str = None) -> Dict[str, Any]:
+        """Create standardized failed result"""
+        failed_result = {
+            "query": query,
+            "category": category,
+            
+            # Metric 1: Total Latency - Always measurable even on failure
+            "latency_metrics": {
+                "total_latency": total_time,
+                "timing_details": timing_details,
+                "meets_target": total_time <= 60.0
+            },
+            
+            # Metric 2: Condition Extraction - Partial data may be available before failure
+            "extraction_metrics": {
+                "extraction_success": False,
+                "extracted_condition": condition_result.get('condition') if condition_result else None,
+                "query_status": condition_result.get('query_status') if condition_result else status,
+                "extraction_time": timing_details.get('step1_condition_extraction', 0.0)
+            },
+            
+            # Metric 3: Retrieval Relevance - Failed due to pipeline failure
+            "relevance_metrics": {
+                "average_relevance": 0.0,
+                "retrieved_count": 0,
+                "meets_threshold": False,
+                "retrieval_time": timing_details.get('step3_retrieval', 0.0)
+            },
+            
+            # Metric 4: Retrieval Coverage - Failed due to pipeline failure  
+            "coverage_metrics": {
+                "coverage_score": 0.0,
+                "meets_threshold": False,
+                "generation_time": timing_details.get('step4_generation', 0.0)
+            },
+            
+            # Note: Metrics 5-6 (Clinical Actionability & Evidence Quality) 
+            # are collected by metric5_6_llm_judge_evaluator.py using medical_outputs
+            # Metrics 7-8 (Precision@K & MRR) are collected by metric7_8_precision_MRR.py 
+            # using comprehensive_details pipeline data
+            
+            "overall_success": False,
+            "status": status,
+            "error": error,
+            "timestamp": datetime.now().isoformat()
+        }
+        
+        # For failed results, precision/MRR analysis data is not ready
+        failed_result['precision_mrr_ready'] = False
+        
+        self.comprehensive_results.append(failed_result)
+        return failed_result
+    
+    def _detect_query_intention(self, query: str) -> str:
+        """Simplified query intention detection (from app.py)"""
+        query_lower = query.lower()
+        
+        if any(word in query_lower for word in ['diagnos', 'differential', 'possible', 'causes']):
+            return 'diagnosis'
+        elif any(word in query_lower for word in ['treat', 'manage', 'therapy', 'intervention']):
+            return 'treatment'
+        else:
+            return 'mixed'    
+    def parse_queries_from_file(self, filepath: str) -> Dict[str, List[Dict]]:
+        """Parse queries from file with category labels"""
+        print(f"📁 Reading queries from file: {filepath}")
+        
+        try:
+            with open(filepath, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            # Parse queries with category labels
+            queries_by_category = {
+                "diagnosis": [],
+                "treatment": [], 
+                "mixed": []
+            }
+            
+            lines = content.strip().split('\n')
+            
+            for line in lines:
+                line = line.strip()
+                if not line:
+                    continue
+                
+                # Parse format: "1.diagnosis: query text"
+                match = re.match(r'^\d+\.(diagnosis|treatment|mixed/complicated|mixed):\s*(.+)', line, re.IGNORECASE)
+                if match:
+                    category_raw = match.group(1).lower()
+                    query_text = match.group(2).strip()
+                    
+                    # Normalize category name
+                    if category_raw in ['mixed/complicated', 'mixed']:
+                        category = 'mixed'
+                    else:
+                        category = category_raw
+                    
+                    if category in queries_by_category and len(query_text) > 15:
+                        queries_by_category[category].append({
+                            "text": query_text,
+                            "category": category
+                        })
+            
+            print(f"📋 Parsed queries by category:")
+            for category, category_queries in queries_by_category.items():
+                print(f"  {category.capitalize()}: {len(category_queries)} queries")
+            
+            return queries_by_category
+            
+        except Exception as e:
+            print(f"❌ Failed to read file: {e}")
+            return {"error": f"Failed to read file: {e}"}
+    
+    def calculate_metric_statistics(self, metric_name: str) -> Dict[str, Any]:
+        """Calculate statistics for a specific metric across all results"""
+        category_stats = {}
+        all_successful_results = []
+        
+        # Group results by category
+        results_by_category = {
+            "diagnosis": [],
+            "treatment": [],
+            "mixed": []
+        }
+        
+        for result in self.comprehensive_results:
+            category = result.get('category', 'unknown')
+            if category in results_by_category:
+                results_by_category[category].append(result)
+                if result.get('overall_success'):
+                    all_successful_results.append(result)
+        
+        # Calculate statistics for each category based on metric type
+        for category, results in results_by_category.items():
+            successful_results = [r for r in results if r.get('overall_success')]
+            
+            if metric_name == "latency":
+                if successful_results:
+                    latencies = [r['latency_metrics']['total_latency'] for r in successful_results]
+                    category_stats[category] = {
+                        "average_latency": sum(latencies) / len(latencies),
+                        "std_deviation": self._calculate_std(latencies),
+                        "min_latency": min(latencies),
+                        "max_latency": max(latencies),
+                        "query_count": len(latencies),
+                        "target_compliance": sum(1 for lat in latencies if lat <= 60.0) / len(latencies),
+                        "individual_latencies": latencies
+                    }
+                else:
+                    category_stats[category] = self._get_empty_latency_stats()
+            
+            elif metric_name == "extraction":
+                extraction_successes = [r['extraction_metrics']['extraction_success'] for r in results]
+                successful_extractions = sum(extraction_successes)
+                
+                category_stats[category] = {
+                    "success_rate": successful_extractions / len(results) if results else 0.0,
+                    "successful_count": successful_extractions,
+                    "total_count": len(results),
+                    "average_extraction_time": sum(r['extraction_metrics']['extraction_time'] for r in results) / len(results) if results else 0.0,
+                    "meets_threshold": (successful_extractions / len(results)) >= 0.8 if results else False
+                }
+            
+            elif metric_name == "relevance":
+                if successful_results:
+                    relevance_scores = [r['relevance_metrics']['average_relevance'] for r in successful_results]
+                    category_stats[category] = {
+                        "average_relevance": sum(relevance_scores) / len(relevance_scores),
+                        "max_relevance": max(relevance_scores),
+                        "min_relevance": min(relevance_scores),
+                        "successful_retrievals": len(successful_results),
+                        "total_queries": len(results),
+                        "meets_threshold": (sum(relevance_scores) / len(relevance_scores)) >= 0.85,
+                        "individual_relevance_scores": relevance_scores
+                    }
+                else:
+                    category_stats[category] = self._get_empty_relevance_stats(len(results))
+            
+            elif metric_name == "coverage":
+                if successful_results:
+                    coverage_scores = [r['coverage_metrics']['coverage_score'] for r in successful_results]
+                    category_stats[category] = {
+                        "average_coverage": sum(coverage_scores) / len(coverage_scores),
+                        "max_coverage": max(coverage_scores),
+                        "min_coverage": min(coverage_scores),
+                        "successful_evaluations": len(successful_results),
+                        "total_queries": len(results),
+                        "meets_threshold": (sum(coverage_scores) / len(coverage_scores)) >= 0.4,
+                        "individual_coverage_scores": coverage_scores
+                    }
+                else:
+                    category_stats[category] = self._get_empty_coverage_stats(len(results))
+        
+        # Calculate overall statistics
+        overall_stats = self._calculate_overall_stats(metric_name, all_successful_results)
+        
+        return {
+            "category_results": category_stats,
+            "overall_results": overall_stats,
+            "timestamp": datetime.now().isoformat()
+        }
+    
+    def _calculate_std(self, values: List[float]) -> float:
+        """Calculate standard deviation"""
+        if len(values) < 2:
+            return 0.0
+        
+        mean = sum(values) / len(values)
+        variance = sum((x - mean) ** 2 for x in values) / len(values)
+        return variance ** 0.5
+    
+    def _get_empty_latency_stats(self) -> Dict[str, Any]:
+        """Return empty latency statistics"""
+        return {
+            "average_latency": 0.0,
+            "std_deviation": 0.0,
+            "min_latency": 0.0,
+            "max_latency": 0.0,
+            "query_count": 0,
+            "target_compliance": 0.0,
+            "individual_latencies": []
+        }
+    
+    def _get_empty_relevance_stats(self, total_queries: int) -> Dict[str, Any]:
+        """Return empty relevance statistics"""
+        return {
+            "average_relevance": 0.0,
+            "max_relevance": 0.0,
+            "min_relevance": 0.0,
+            "successful_retrievals": 0,
+            "total_queries": total_queries,
+            "meets_threshold": False,
+            "individual_relevance_scores": []
+        }
+    
+    def _get_empty_coverage_stats(self, total_queries: int) -> Dict[str, Any]:
+        """Return empty coverage statistics"""
+        return {
+            "average_coverage": 0.0,
+            "max_coverage": 0.0,
+            "min_coverage": 0.0,
+            "successful_evaluations": 0,
+            "total_queries": total_queries,
+            "meets_threshold": False,
+            "individual_coverage_scores": []
+        }
+    
+    def _calculate_overall_stats(self, metric_name: str, all_successful_results: List[Dict]) -> Dict[str, Any]:
+        """Calculate overall statistics for a specific metric"""
+        total_queries = len(self.comprehensive_results)
+        
+        if metric_name == "latency" and all_successful_results:
+            latencies = [r['latency_metrics']['total_latency'] for r in all_successful_results]
+            return {
+                "average_latency": sum(latencies) / len(latencies),
+                "std_deviation": self._calculate_std(latencies),
+                "min_latency": min(latencies),
+                "max_latency": max(latencies),
+                "successful_queries": len(all_successful_results),
+                "total_queries": total_queries,
+                "target_compliance": sum(1 for lat in latencies if lat <= 60.0) / len(latencies)
+            }
+        
+        elif metric_name == "extraction":
+            all_extractions = [r['extraction_metrics']['extraction_success'] for r in self.comprehensive_results]
+            successful_extractions = sum(all_extractions)
+            return {
+                "success_rate": successful_extractions / len(all_extractions) if all_extractions else 0.0,
+                "successful_count": successful_extractions,
+                "total_count": len(all_extractions),
+                "target_compliance": (successful_extractions / len(all_extractions)) >= 0.8 if all_extractions else False
+            }
+        
+        elif metric_name == "relevance" and all_successful_results:
+            relevance_scores = [r['relevance_metrics']['average_relevance'] for r in all_successful_results]
+            return {
+                "average_relevance": sum(relevance_scores) / len(relevance_scores),
+                "max_relevance": max(relevance_scores),
+                "min_relevance": min(relevance_scores),
+                "successful_queries": len(all_successful_results),
+                "total_queries": total_queries,
+                "meets_threshold": (sum(relevance_scores) / len(relevance_scores)) >= 0.85,
+                "target_compliance": (sum(relevance_scores) / len(relevance_scores)) >= 0.7
+            }
+        
+        elif metric_name == "coverage" and all_successful_results:
+            coverage_scores = [r['coverage_metrics']['coverage_score'] for r in all_successful_results]
+            return {
+                "average_coverage": sum(coverage_scores) / len(coverage_scores),
+                "max_coverage": max(coverage_scores),
+                "min_coverage": min(coverage_scores),
+                "successful_queries": len(all_successful_results),
+                "total_queries": total_queries,
+                "meets_threshold": (sum(coverage_scores) / len(coverage_scores)) >= 0.4
+            }
+        
+        # Return empty stats for failed cases
+        return {
+            "average_value": 0.0,
+            "successful_queries": len(all_successful_results),
+            "total_queries": total_queries,
+            "meets_threshold": False
+        }    
+    def save_all_metric_statistics(self) -> Dict[str, str]:
+        """Save separate statistics files for each metric"""
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        saved_files = {}
+        
+        # Save statistics for each metric
+        for metric_name in ["latency", "extraction", "relevance", "coverage"]:
+            stats = self.calculate_metric_statistics(metric_name)
+            filename = f"{metric_name}_statistics_{timestamp}.json"
+            filepath = results_dir / filename
+            
+            with open(filepath, 'w', encoding='utf-8') as f:
+                json.dump(stats, f, indent=2, ensure_ascii=False)
+            
+            saved_files[metric_name] = str(filepath)
+            print(f"📊 {metric_name.capitalize()} statistics saved to: {filepath}")
+        
+        return saved_files
+    
+    def save_medical_outputs(self, filename: str = None) -> str:
+        """Save medical advice outputs for model comparison"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"medical_outputs_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        # Create comprehensive output data
+        output_data = {
+            "evaluation_metadata": {
+                "total_outputs": len(self.medical_outputs),
+                "categories": list(set(output['category'] for output in self.medical_outputs)),
+                "timestamp": datetime.now().isoformat(),
+                "model_type": "Med42-70B_RAG_enhanced"  # For future comparison
+            },
+            "medical_outputs": self.medical_outputs
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(output_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"📝 Medical outputs saved to: {filepath}")
+        return str(filepath)
+    
+    def save_comprehensive_details(self, filename: str = None) -> str:
+        """Save comprehensive detailed results"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"comprehensive_details_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        # Create comprehensive evaluation data
+        comprehensive_data = {
+            "evaluation_metadata": {
+                "total_queries": len(self.comprehensive_results),
+                "successful_queries": len([r for r in self.comprehensive_results if r.get('overall_success')]),
+                "timestamp": datetime.now().isoformat(),
+                "evaluator_type": "comprehensive_metrics_1_to_4",
+                "metrics_evaluated": ["latency", "extraction", "relevance", "coverage"]
+            },
+            "comprehensive_results": self.comprehensive_results
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(comprehensive_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"📋 Comprehensive details saved to: {filepath}")
+        return str(filepath)
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent comprehensive evaluation interface"""
+    
+    print("🚀 OnCall.ai Comprehensive Evaluator - Metrics 1-4 in Single Run")
+    
+    if len(sys.argv) > 1:
+        query_file = sys.argv[1]
+    else:
+        # Default to evaluation/single_test_query.txt for initial testing
+        query_file = Path(__file__).parent / "single_test_query.txt"
+    
+    if not os.path.exists(query_file):
+        print(f"❌ Query file not found: {query_file}")
+        print("Usage: python latency_evaluator.py [query_file.txt]")
+        sys.exit(1)
+    
+    # Initialize evaluator
+    evaluator = ComprehensiveEvaluator()
+    
+    # Parse queries from file
+    queries_by_category = evaluator.parse_queries_from_file(str(query_file))
+    
+    if "error" in queries_by_category:
+        print(f"❌ Failed to parse queries: {queries_by_category['error']}")
+        sys.exit(1)
+    
+    # Test each query comprehensively
+    print(f"\n🧪 Comprehensive Evaluation - All Metrics in Single Run")
+    print(f"📊 Collecting metrics 1-4 from single app.py pipeline execution")
+    
+    for category, queries in queries_by_category.items():
+        if not queries:
+            continue
+            
+        print(f"\n📂 Testing {category.upper()} queries:")
+        
+        for i, query_info in enumerate(queries):
+            query_text = query_info['text']
+            print(f"\n🔍 Query {i+1}/{len(queries)} in {category} category:")
+            print(f"   Text: {query_text}")
+            
+            # Comprehensive evaluation (collects all metrics 1-4)
+            result = evaluator.evaluate_single_query_comprehensive(query_text, category)
+            
+            # Pause between queries to avoid rate limits
+            if i < len(queries) - 1:
+                print(f"   ⏳ Pausing 5s before next query...")
+                time.sleep(5)
+        
+        # Longer pause between categories
+        if category != list(queries_by_category.keys())[-1]:
+            print(f"\n⏳ Pausing 10s before next category...")
+            time.sleep(10)
+    
+    # Generate and save all metric statistics
+    print(f"\n📊 Generating comprehensive analysis for all metrics...")
+    
+    # Save separate statistics for each metric
+    saved_stats = evaluator.save_all_metric_statistics()
+    
+    # Save medical outputs for model comparison
+    outputs_path = evaluator.save_medical_outputs()
+    
+    # Save comprehensive details
+    details_path = evaluator.save_comprehensive_details()
+    
+    # Print comprehensive summary
+    print(f"\n📊 === COMPREHENSIVE EVALUATION SUMMARY ===")
+    
+    for metric_name in ["latency", "extraction", "relevance", "coverage"]:
+        stats = evaluator.calculate_metric_statistics(metric_name)
+        overall_results = stats['overall_results']
+        
+        print(f"\n{metric_name.upper()} METRICS:")
+        
+        if metric_name == "latency":
+            print(f"   Average: {overall_results['average_latency']:.2f}s (±{overall_results['std_deviation']:.2f})")
+            print(f"   60s Target: {'✅ Met' if overall_results['target_compliance'] >= 0.8 else '❌ Not Met'}")
+        
+        elif metric_name == "extraction":
+            print(f"   Success Rate: {overall_results['success_rate']:.1%}")
+            print(f"   80% Target: {'✅ Met' if overall_results['target_compliance'] else '❌ Not Met'}")
+        
+        elif metric_name == "relevance":
+            print(f"   Average Relevance: {overall_results['average_relevance']:.3f}")
+            print(f"   0.70 Target: {'✅ Met' if overall_results.get('target_compliance', False) else '❌ Not Met'}")
+        
+        elif metric_name == "coverage":
+            print(f"   Average Coverage: {overall_results['average_coverage']:.3f} ({overall_results['average_coverage']*100:.1f}%)")
+            print(f"   40% Target: {'✅ Met' if overall_results['meets_threshold'] else '❌ Not Met'}")
+    
+    print(f"\n✅ Comprehensive evaluation complete! Files saved:")
+    for metric_name, filepath in saved_stats.items():
+        print(f"   📊 {metric_name.capitalize()}: {filepath}")
+    print(f"   📝 Medical Outputs: {outputs_path}")
+    print(f"   📋 Comprehensive Details: {details_path}")
+    print(f"\n💡 Next step: Run downstream evaluators for metrics 5-8")
+    print(f"   python metric5_6_llm_judge_evaluator.py rag")
+    print(f"   python metric7_8_precision_MRR.py {details_path}")
+    print(f"   python latency_chart_generator.py")
+    print(f"   python extraction_chart_generator.py  # (create separately)")
+    print(f"   python relevance_chart_generator.py   # (create separately)")
+    print(f"   python coverage_chart_generator.py    # (create separately)")

evaluation/metric1_latency_chart_generator.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Latency Chart Generator
+==========================================
+
+Generates comprehensive latency analysis charts from saved statistics.
+Reads JSON files produced by latency_evaluator.py and creates visualizations.
+
+No LLM calls - pure data visualization.
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any
+from datetime import datetime
+from pathlib import Path
+import glob
+
+# Visualization imports
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+import numpy as np
+
+
+class LatencyChartGenerator:
+    """Generate charts from latency evaluation statistics - no LLM dependency"""
+    
+    def __init__(self):
+        """Initialize chart generator"""
+        print("📈 Initializing Latency Chart Generator...")
+        
+        # Set up professional chart style
+        plt.style.use('default')
+        sns.set_palette("husl")
+        
+        print("✅ Chart Generator ready")
+    
+    def load_latest_statistics(self, results_dir: str = None) -> Dict[str, Any]:
+        """
+        Load the most recent latency statistics file
+        
+        Args:
+            results_dir: Directory containing statistics files
+        """
+        if results_dir is None:
+            results_dir = Path(__file__).parent / "results"
+        
+        # Find latest statistics file
+        pattern = str(results_dir / "latency_statistics_*.json")
+        stat_files = glob.glob(pattern)
+        
+        if not stat_files:
+            raise FileNotFoundError(f"No latency statistics files found in {results_dir}")
+        
+        # Get the most recent file
+        latest_file = max(stat_files, key=os.path.getmtime)
+        
+        print(f"📊 Loading statistics from: {latest_file}")
+        
+        with open(latest_file, 'r', encoding='utf-8') as f:
+            stats = json.load(f)
+        
+        return stats
+    
+    def generate_comprehensive_charts(self, stats: Dict[str, Any]) -> str:
+        """
+        Generate comprehensive 4-category latency analysis charts
+        
+        Creates professional charts showing:
+        1. Category comparison bar chart
+        2. Individual query scatter plot  
+        3. Statistical summary table
+        4. Performance distribution box plot
+        """
+        try:
+            # Create figure with subplots
+            fig, axes = plt.subplots(2, 2, figsize=(16, 12))
+            fig.suptitle('OnCall.ai Latency Analysis - Category Comparison', 
+                        fontsize=16, fontweight='bold')
+            
+            category_results = stats['category_results']
+            overall_results = stats['overall_results']
+            
+            # Chart 1: Category Comparison Bar Chart
+            ax1 = axes[0, 0]
+            categories = []
+            avg_latencies = []
+            std_devs = []
+            
+            # Collect category data
+            for category, cat_stats in category_results.items():
+                if cat_stats['query_count'] > 0:
+                    categories.append(category.replace('_', ' ').title())
+                    avg_latencies.append(cat_stats['average_latency'])
+                    std_devs.append(cat_stats['std_deviation'])
+            
+            # Add overall
+            categories.append('Overall')
+            avg_latencies.append(overall_results['average_latency'])
+            std_devs.append(overall_results['std_deviation'])
+            
+            # Create bar chart with error bars
+            bars = ax1.bar(categories, avg_latencies, capsize=5, alpha=0.8, 
+                          color=['#1f77b4', '#ff7f0e', '#d62728', '#2ca02c'])
+            ax1.errorbar(categories, avg_latencies, yerr=std_devs, fmt='none', 
+                        color='black', capsize=3, capthick=1)
+            
+            ax1.set_title('Average Latency by Category', fontweight='bold')
+            ax1.set_ylabel('Latency (seconds)')
+            ax1.set_xlabel('Query Category')
+            ax1.grid(True, alpha=0.3)
+            
+            # Add value labels on bars
+            for bar, avg, std in zip(bars, avg_latencies, std_devs):
+                height = bar.get_height()
+                ax1.text(bar.get_x() + bar.get_width()/2., height + std*0.1,
+                        f'{avg:.1f}s', ha='center', va='bottom', fontweight='bold')
+            
+            # Add target line
+            ax1.axhline(y=30.0, color='red', linestyle='--', alpha=0.7, label='30s Target')
+            ax1.legend()
+            
+            # Chart 2: Individual Query Performance
+            ax2 = axes[0, 1]
+            
+            query_indices = []
+            latencies = []
+            colors = []
+            
+            color_map = {'diagnosis': '#1f77b4', 'treatment': '#ff7f0e', 'mixed': '#d62728'}
+            query_idx = 0
+            
+            for category, cat_stats in category_results.items():
+                for latency in cat_stats['individual_latencies']:
+                    query_indices.append(query_idx)
+                    latencies.append(latency)
+                    colors.append(color_map.get(category, 'gray'))
+                    query_idx += 1
+            
+            if latencies:
+                ax2.scatter(query_indices, latencies, c=colors, alpha=0.7, s=100)
+                ax2.set_title('Individual Query Performance', fontweight='bold')
+                ax2.set_ylabel('Latency (seconds)')
+                ax2.set_xlabel('Query Index')
+                ax2.grid(True, alpha=0.3)
+                
+                # Add target line
+                ax2.axhline(y=30.0, color='red', linestyle='--', alpha=0.7, label='30s Target')
+                
+                # Add category legend
+                from matplotlib.patches import Patch
+                legend_elements = [Patch(facecolor=color_map[cat], label=cat.title()) 
+                                 for cat in color_map.keys() if cat in category_results.keys()]
+                ax2.legend(handles=legend_elements)
+            else:
+                ax2.text(0.5, 0.5, 'No latency data available', 
+                        ha='center', va='center', transform=ax2.transAxes)
+                ax2.set_title('Individual Query Performance', fontweight='bold')
+            
+            # Chart 3: Statistical Summary Table
+            ax3 = axes[1, 0]
+            ax3.axis('tight')
+            ax3.axis('off')
+            
+            # Create summary table
+            table_data = []
+            headers = ['Category', 'Avg (s)', 'Std (s)', 'Min (s)', 'Max (s)', 'Count']
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats['query_count'] > 0:
+                    table_data.append([
+                        category.replace('_', ' ').title(),
+                        f"{cat_stats['average_latency']:.2f}",
+                        f"{cat_stats['std_deviation']:.2f}",
+                        f"{cat_stats['min_latency']:.2f}",
+                        f"{cat_stats['max_latency']:.2f}",
+                        str(cat_stats['query_count'])
+                    ])
+            
+            # Add overall row
+            table_data.append([
+                'Overall',
+                f"{overall_results['average_latency']:.2f}",
+                f"{overall_results['std_deviation']:.2f}",
+                f"{overall_results['min_latency']:.2f}",
+                f"{overall_results['max_latency']:.2f}",
+                str(overall_results['successful_queries'])
+            ])
+            
+            if table_data:
+                table = ax3.table(cellText=table_data, colLabels=headers,
+                                cellLoc='center', loc='center',
+                                colWidths=[0.2, 0.15, 0.15, 0.15, 0.15, 0.1])
+                table.auto_set_font_size(False)
+                table.set_fontsize(10)
+                table.scale(1, 2)
+                
+                # Style the table header
+                for i in range(len(headers)):
+                    table[(0, i)].set_text_props(weight='bold', color='white')
+                    table[(0, i)].set_facecolor('#2E7D32')
+            
+            ax3.set_title('Statistical Summary', fontweight='bold', pad=20)
+            
+            # Chart 4: Performance Distribution
+            ax4 = axes[1, 1]
+            
+            # Create box plot if we have multiple data points
+            box_data = []
+            box_labels = []
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats['individual_latencies'] and len(cat_stats['individual_latencies']) > 0:
+                    box_data.append(cat_stats['individual_latencies'])
+                    box_labels.append(category.replace('_', ' ').title())
+            
+            if box_data and len(box_data) > 0:
+                box_plot = ax4.boxplot(box_data, labels=box_labels, patch_artist=True)
+                
+                # Color the boxes
+                colors = ['#1f77b4', '#ff7f0e', '#d62728']
+                for patch, color in zip(box_plot['boxes'], colors[:len(box_plot['boxes'])]):
+                    patch.set_facecolor(color)
+                    patch.set_alpha(0.7)
+                
+                ax4.set_title('Latency Distribution by Category', fontweight='bold')
+                ax4.set_ylabel('Latency (seconds)')
+                ax4.grid(True, alpha=0.3)
+                
+                # Add target line
+                ax4.axhline(y=30.0, color='red', linestyle='--', alpha=0.7, label='30s Target')
+                ax4.legend()
+            else:
+                # For single data points, show a simple bar chart
+                single_categories = []
+                single_latencies = []
+                
+                for category, cat_stats in category_results.items():
+                    if cat_stats['query_count'] > 0:
+                        single_categories.append(category.replace('_', ' ').title())
+                        single_latencies.append(cat_stats['average_latency'])
+                
+                if single_categories:
+                    ax4.bar(single_categories, single_latencies, alpha=0.7, 
+                           color=['#1f77b4', '#ff7f0e', '#d62728'][:len(single_categories)])
+                    ax4.set_title('Category Latency (Single Query Each)', fontweight='bold')
+                    ax4.set_ylabel('Latency (seconds)')
+                    ax4.grid(True, alpha=0.3)
+                    ax4.axhline(y=30.0, color='red', linestyle='--', alpha=0.7, label='30s Target')
+                    ax4.legend()
+                else:
+                    ax4.text(0.5, 0.5, 'No data available for distribution plot', 
+                            ha='center', va='center', transform=ax4.transAxes)
+                    ax4.set_title('Latency Distribution', fontweight='bold')
+            
+            # Adjust layout and save
+            plt.tight_layout()
+            
+            # Save chart
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            chart_filename = f"latency_analysis_charts_{timestamp}.png"
+            
+            # Ensure results directory exists
+            results_dir = Path(__file__).parent / "results"
+            results_dir.mkdir(exist_ok=True)
+            chart_path = results_dir / chart_filename
+            
+            plt.savefig(chart_path, dpi=300, bbox_inches='tight', 
+                       facecolor='white', edgecolor='none')
+            plt.close()
+            
+            print(f"📈 Charts saved to: {chart_path}")
+            return str(chart_path)
+            
+        except Exception as e:
+            print(f"❌ Chart generation failed: {e}")
+            return ""
+    
+    def print_statistics_summary(self, stats: Dict[str, Any]):
+        """Print formatted statistics summary to console"""
+        category_results = stats['category_results']
+        overall_results = stats['overall_results']
+        
+        print(f"\n📊 === LATENCY ANALYSIS CHART SUMMARY ===")
+        print(f"Overall Performance:")
+        print(f"   Average Latency: {overall_results['average_latency']:.2f}s (±{overall_results['std_deviation']:.2f})")
+        print(f"   Success Rate: {overall_results['successful_queries']}/{overall_results['total_queries']}")
+        print(f"   30s Target Compliance: {overall_results['target_compliance']:.1%}")
+        
+        print(f"\nCategory Breakdown:")
+        for category, cat_stats in category_results.items():
+            if cat_stats['query_count'] > 0:
+                print(f"   {category.capitalize()}: {cat_stats['average_latency']:.2f}s (±{cat_stats['std_deviation']:.2f}) [{cat_stats['query_count']} queries]")
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent chart generation interface"""
+    
+    print("📈 OnCall.ai Latency Chart Generator")
+    
+    # Initialize chart generator
+    chart_gen = LatencyChartGenerator()
+    
+    try:
+        # Load latest statistics
+        stats = chart_gen.load_latest_statistics()
+        
+        # Generate charts
+        chart_path = chart_gen.generate_comprehensive_charts(stats)
+        
+        # Print summary
+        chart_gen.print_statistics_summary(stats)
+        
+        print(f"\n✅ Chart generation complete!")
+        print(f"📈 Charts saved to: {chart_path}")
+        
+    except FileNotFoundError as e:
+        print(f"❌ {e}")
+        print("💡 Please run latency_evaluator.py first to generate statistics data")
+    except Exception as e:
+        print(f"❌ Chart generation failed: {e}")

evaluation/metric2_extraction_chart_generator.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Extraction Chart Generator
+============================================
+
+Generates extraction success rate charts from saved statistics.
+Reads JSON files produced by comprehensive evaluator.
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any
+from datetime import datetime
+from pathlib import Path
+import glob
+
+# Visualization imports
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+import numpy as np
+
+
+class ExtractionChartGenerator:
+    """Generate charts for condition extraction metrics"""
+    
+    def __init__(self):
+        """Initialize chart generator"""
+        print("📈 Initializing Extraction Chart Generator...")
+        plt.style.use('default')
+        sns.set_palette("husl")
+        print("✅ Chart Generator ready")
+    
+    def load_latest_extraction_statistics(self, results_dir: str = None) -> Dict[str, Any]:
+        """Load the most recent extraction statistics file"""
+        if results_dir is None:
+            results_dir = Path(__file__).parent / "results"
+        
+        pattern = str(results_dir / "extraction_statistics_*.json")
+        stat_files = glob.glob(pattern)
+        
+        if not stat_files:
+            raise FileNotFoundError(f"No extraction statistics files found in {results_dir}")
+        
+        latest_file = max(stat_files, key=os.path.getmtime)
+        print(f"📊 Loading extraction statistics from: {latest_file}")
+        
+        with open(latest_file, 'r', encoding='utf-8') as f:
+            stats = json.load(f)
+        
+        return stats
+    
+    def generate_extraction_charts(self, stats: Dict[str, Any]) -> str:
+        """Generate extraction success rate analysis charts"""
+        try:
+            fig, axes = plt.subplots(2, 2, figsize=(16, 12))
+            fig.suptitle('OnCall.ai Extraction Success Rate Analysis', fontsize=16, fontweight='bold')
+            
+            category_results = stats['category_results']
+            overall_results = stats['overall_results']
+            
+            # Chart 1: Success Rate by Category
+            ax1 = axes[0, 0]
+            categories = []
+            success_rates = []
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats['total_count'] > 0:
+                    categories.append(category.replace('_', ' ').title())
+                    success_rates.append(cat_stats['success_rate'] * 100)
+            
+            categories.append('Overall')
+            success_rates.append(overall_results['success_rate'] * 100)
+            
+            bars = ax1.bar(categories, success_rates, alpha=0.8, color=['#1f77b4', '#ff7f0e', '#d62728', '#2ca02c'])
+            ax1.set_title('Extraction Success Rate by Category', fontweight='bold')
+            ax1.set_ylabel('Success Rate (%)')
+            ax1.set_xlabel('Query Category')
+            ax1.grid(True, alpha=0.3)
+            
+            # Add target line
+            ax1.axhline(y=80, color='red', linestyle='--', alpha=0.7, label='80% Target')
+            ax1.legend()
+            
+            # Add value labels
+            for bar, rate in zip(bars, success_rates):
+                height = bar.get_height()
+                ax1.text(bar.get_x() + bar.get_width()/2., height + 1,
+                        f'{rate:.1f}%', ha='center', va='bottom', fontweight='bold')
+            
+            # Chart 2: Success Count
+            ax2 = axes[0, 1]
+            successful_counts = []
+            total_counts = []
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats['total_count'] > 0:
+                    successful_counts.append(cat_stats['successful_count'])
+                    total_counts.append(cat_stats['total_count'])
+            
+            successful_counts.append(overall_results['successful_count'])
+            total_counts.append(overall_results['total_count'])
+            
+            x = np.arange(len(categories))
+            width = 0.35
+            
+            ax2.bar(x - width/2, successful_counts, width, label='Successful', alpha=0.8)
+            ax2.bar(x + width/2, total_counts, width, label='Total', alpha=0.8)
+            
+            ax2.set_title('Extraction Success Count', fontweight='bold')
+            ax2.set_ylabel('Query Count')
+            ax2.set_xlabel('Query Category')
+            ax2.set_xticks(x)
+            ax2.set_xticklabels(categories)
+            ax2.legend()
+            ax2.grid(True, alpha=0.3)
+            
+            # Chart 3: Statistical Summary Table
+            ax3 = axes[1, 0]
+            ax3.axis('tight')
+            ax3.axis('off')
+            
+            table_data = []
+            headers = ['Category', 'Success Rate', 'Success/Total', 'Avg Time (s)', 'Target Met']
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats['total_count'] > 0:
+                    table_data.append([
+                        category.replace('_', ' ').title(),
+                        f"{cat_stats['success_rate']:.1%}",
+                        f"{cat_stats['successful_count']}/{cat_stats['total_count']}",
+                        f"{cat_stats['average_extraction_time']:.3f}",
+                        '✅' if cat_stats.get('meets_threshold', False) else '❌'
+                    ])
+            
+            table_data.append([
+                'Overall',
+                f"{overall_results['success_rate']:.1%}",
+                f"{overall_results['successful_count']}/{overall_results['total_count']}",
+                '-',
+                '✅' if overall_results.get('target_compliance', False) else '❌'
+            ])
+            
+            if table_data:
+                table = ax3.table(cellText=table_data, colLabels=headers,
+                                cellLoc='center', loc='center')
+                table.auto_set_font_size(False)
+                table.set_fontsize(10)
+                table.scale(1, 2)
+                
+                # Style header
+                for i in range(len(headers)):
+                    table[(0, i)].set_text_props(weight='bold', color='white')
+                    table[(0, i)].set_facecolor('#2E7D32')
+            
+            ax3.set_title('Extraction Statistics Summary', fontweight='bold', pad=20)
+            
+            # Chart 4: Performance visualization
+            ax4 = axes[1, 1]
+            
+            # Simple performance indicator
+            overall_rate = overall_results['success_rate'] * 100
+            colors = ['#d62728' if overall_rate < 80 else '#2ca02c']
+            
+            wedges, texts, autotexts = ax4.pie([overall_rate, 100-overall_rate], 
+                                              labels=['Successful', 'Failed'],
+                                              autopct='%1.1f%%',
+                                              colors=['#2ca02c', '#ffcccc'],
+                                              startangle=90)
+            
+            ax4.set_title(f'Overall Extraction Success\n{overall_rate:.1f}% Success Rate', fontweight='bold')
+            
+            plt.tight_layout()
+            
+            # Save chart
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            chart_filename = f"extraction_analysis_charts_{timestamp}.png"
+            
+            results_dir = Path(__file__).parent / "results"
+            results_dir.mkdir(exist_ok=True)
+            chart_path = results_dir / chart_filename
+            
+            plt.savefig(chart_path, dpi=300, bbox_inches='tight', facecolor='white')
+            plt.close()
+            
+            print(f"📈 Extraction charts saved to: {chart_path}")
+            return str(chart_path)
+            
+        except Exception as e:
+            print(f"❌ Extraction chart generation failed: {e}")
+            return ""
+
+
+if __name__ == "__main__":
+    """Independent extraction chart generation"""
+    
+    print("📈 OnCall.ai Extraction Chart Generator")
+    
+    chart_gen = ExtractionChartGenerator()
+    
+    try:
+        stats = chart_gen.load_latest_extraction_statistics()
+        chart_path = chart_gen.generate_extraction_charts(stats)
+        
+        print(f"\n✅ Extraction chart generation complete!")
+        print(f"📈 Charts saved to: {chart_path}")
+        
+    except FileNotFoundError as e:
+        print(f"❌ {e}")
+        print("💡 Please run latency_evaluator.py first to generate extraction statistics data")
+    except Exception as e:
+        print(f"❌ Chart generation failed: {e}")

evaluation/metric3_relevance_chart_generator.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Relevance Chart Generator
+============================================
+
+Generates retrieval relevance charts from saved statistics.
+Shows cosine similarity analysis and threshold compliance.
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any
+from datetime import datetime
+from pathlib import Path
+import glob
+
+# Visualization imports
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+import numpy as np
+
+
+class RelevanceChartGenerator:
+    """Generate charts for retrieval relevance metrics"""
+    
+    def __init__(self):
+        """Initialize chart generator"""
+        print("📈 Initializing Relevance Chart Generator...")
+        plt.style.use('default')
+        sns.set_palette("husl")
+        print("✅ Chart Generator ready")
+    
+    def load_latest_relevance_statistics(self, results_dir: str = None) -> Dict[str, Any]:
+        """Load the most recent relevance statistics file"""
+        if results_dir is None:
+            results_dir = Path(__file__).parent / "results"
+        
+        pattern = str(results_dir / "relevance_statistics_*.json")
+        stat_files = glob.glob(pattern)
+        
+        if not stat_files:
+            raise FileNotFoundError(f"No relevance statistics files found in {results_dir}")
+        
+        latest_file = max(stat_files, key=os.path.getmtime)
+        print(f"📊 Loading relevance statistics from: {latest_file}")
+        
+        with open(latest_file, 'r', encoding='utf-8') as f:
+            stats = json.load(f)
+        
+        return stats
+    
+    def generate_relevance_charts(self, stats: Dict[str, Any]) -> str:
+        """Generate relevance analysis charts"""
+        try:
+            fig, axes = plt.subplots(2, 2, figsize=(16, 12))
+            fig.suptitle('OnCall.ai Retrieval Relevance Analysis', fontsize=16, fontweight='bold')
+            
+            category_results = stats['category_results']
+            overall_results = stats['overall_results']
+            
+            # Chart 1: Average Relevance by Category
+            ax1 = axes[0, 0]
+            categories = []
+            avg_relevances = []
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats['successful_retrievals'] > 0:
+                    categories.append(category.replace('_', ' ').title())
+                    avg_relevances.append(cat_stats['average_relevance'])
+            
+            categories.append('Overall')
+            avg_relevances.append(overall_results['average_relevance'])
+            
+            bars = ax1.bar(categories, avg_relevances, alpha=0.8, color=['#1f77b4', '#ff7f0e', '#d62728', '#2ca02c'])
+            ax1.set_title('Average Relevance Score by Category', fontweight='bold')
+            ax1.set_ylabel('Relevance Score (Cosine Similarity)')
+            ax1.set_xlabel('Query Category')
+            ax1.grid(True, alpha=0.3)
+            
+            # Add threshold lines
+            ax1.axhline(y=0.2, color='orange', linestyle='--', alpha=0.7, label='0.2 Threshold')
+            ax1.axhline(y=0.70, color='red', linestyle='--', alpha=0.7, label='0.70 Target')
+            ax1.legend()
+            
+            # Add value labels
+            for bar, relevance in zip(bars, avg_relevances):
+                height = bar.get_height()
+                ax1.text(bar.get_x() + bar.get_width()/2., height + 0.01,
+                        f'{relevance:.3f}', ha='center', va='bottom', fontweight='bold')
+            
+            # Chart 2: Relevance Distribution
+            ax2 = axes[0, 1]
+            
+            # Collect all individual relevance scores
+            all_scores = []
+            category_labels = []
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats.get('individual_relevance_scores'):
+                    all_scores.extend(cat_stats['individual_relevance_scores'])
+                    category_labels.extend([category] * len(cat_stats['individual_relevance_scores']))
+            
+            if all_scores:
+                # Create histogram
+                ax2.hist(all_scores, bins=20, alpha=0.7, color='skyblue', edgecolor='black')
+                ax2.axvline(x=0.2, color='orange', linestyle='--', alpha=0.7, label='0.2 Threshold')
+                ax2.axvline(x=0.70, color='red', linestyle='--', alpha=0.7, label='0.70 Target')
+                ax2.axvline(x=np.mean(all_scores), color='green', linestyle='-', alpha=0.8, label=f'Mean: {np.mean(all_scores):.3f}')
+                
+                ax2.set_title('Relevance Score Distribution', fontweight='bold')
+                ax2.set_xlabel('Relevance Score')
+                ax2.set_ylabel('Frequency')
+                ax2.legend()
+                ax2.grid(True, alpha=0.3)
+            else:
+                ax2.text(0.5, 0.5, 'No relevance data available', ha='center', va='center', transform=ax2.transAxes)
+                ax2.set_title('Relevance Score Distribution', fontweight='bold')
+            
+            # Chart 3: Statistical Summary Table
+            ax3 = axes[1, 0]
+            ax3.axis('tight')
+            ax3.axis('off')
+            
+            table_data = []
+            headers = ['Category', 'Avg Relevance', 'Min/Max', 'Success/Total', 'Threshold Met']
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats['total_queries'] > 0:
+                    table_data.append([
+                        category.replace('_', ' ').title(),
+                        f"{cat_stats['average_relevance']:.3f}",
+                        f"{cat_stats['min_relevance']:.3f}/{cat_stats['max_relevance']:.3f}",
+                        f"{cat_stats['successful_retrievals']}/{cat_stats['total_queries']}",
+                        '✅' if cat_stats.get('meets_threshold', False) else '❌'
+                    ])
+            
+            table_data.append([
+                'Overall',
+                f"{overall_results['average_relevance']:.3f}",
+                f"{overall_results['min_relevance']:.3f}/{overall_results['max_relevance']:.3f}",
+                f"{overall_results['successful_queries']}/{overall_results['total_queries']}",
+                '✅' if overall_results.get('target_compliance', False) else '❌'
+            ])
+            
+            if table_data:
+                table = ax3.table(cellText=table_data, colLabels=headers,
+                                cellLoc='center', loc='center')
+                table.auto_set_font_size(False)
+                table.set_fontsize(10)
+                table.scale(1, 2)
+                
+                # Style header
+                for i in range(len(headers)):
+                    table[(0, i)].set_text_props(weight='bold', color='white')
+                    table[(0, i)].set_facecolor('#2E7D32')
+            
+            ax3.set_title('Relevance Statistics Summary', fontweight='bold', pad=20)
+            
+            # Chart 4: Category Comparison Box Plot
+            ax4 = axes[1, 1]
+            
+            box_data = []
+            box_labels = []
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats.get('individual_relevance_scores'):
+                    box_data.append(cat_stats['individual_relevance_scores'])
+                    box_labels.append(category.replace('_', ' ').title())
+            
+            if box_data:
+                box_plot = ax4.boxplot(box_data, labels=box_labels, patch_artist=True)
+                colors = ['#1f77b4', '#ff7f0e', '#d62728']
+                for patch, color in zip(box_plot['boxes'], colors[:len(box_plot['boxes'])]):
+                    patch.set_facecolor(color)
+                    patch.set_alpha(0.7)
+                
+                ax4.axhline(y=0.2, color='orange', linestyle='--', alpha=0.7, label='0.2 Threshold')
+                ax4.axhline(y=0.70, color='red', linestyle='--', alpha=0.7, label='0.70 Target')
+                ax4.set_title('Relevance Distribution by Category', fontweight='bold')
+                ax4.set_ylabel('Relevance Score')
+                ax4.legend()
+                ax4.grid(True, alpha=0.3)
+            else:
+                ax4.text(0.5, 0.5, 'Insufficient data for box plot', ha='center', va='center', transform=ax4.transAxes)
+                ax4.set_title('Relevance Distribution by Category', fontweight='bold')
+            
+            plt.tight_layout()
+            
+            # Save chart
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            chart_filename = f"relevance_analysis_charts_{timestamp}.png"
+            
+            results_dir = Path(__file__).parent / "results"
+            results_dir.mkdir(exist_ok=True)
+            chart_path = results_dir / chart_filename
+            
+            plt.savefig(chart_path, dpi=300, bbox_inches='tight', facecolor='white')
+            plt.close()
+            
+            print(f"📈 Relevance charts saved to: {chart_path}")
+            return str(chart_path)
+            
+        except Exception as e:
+            print(f"❌ Relevance chart generation failed: {e}")
+            return ""
+
+
+if __name__ == "__main__":
+    """Independent relevance chart generation"""
+    
+    print("📈 OnCall.ai Relevance Chart Generator")
+    
+    chart_gen = RelevanceChartGenerator()
+    
+    try:
+        stats = chart_gen.load_latest_relevance_statistics()
+        chart_path = chart_gen.generate_relevance_charts(stats)
+        
+        print(f"\n✅ Relevance chart generation complete!")
+        print(f"📈 Charts saved to: {chart_path}")
+        
+    except FileNotFoundError as e:
+        print(f"❌ {e}")
+        print("💡 Please run latency_evaluator.py first to generate relevance statistics data")
+    except Exception as e:
+        print(f"❌ Chart generation failed: {e}")

evaluation/metric4_coverage_chart_generator.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Coverage Chart Generator
+===========================================
+
+Generates retrieval coverage charts from saved statistics.
+Shows how well generated advice utilizes retrieved content.
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any
+from datetime import datetime
+from pathlib import Path
+import glob
+
+# Visualization imports
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+import numpy as np
+
+
+class CoverageChartGenerator:
+    """Generate charts for retrieval coverage metrics"""
+    
+    def __init__(self):
+        """Initialize chart generator"""
+        print("📈 Initializing Coverage Chart Generator...")
+        plt.style.use('default')
+        sns.set_palette("husl")
+        print("✅ Chart Generator ready")
+    
+    def load_latest_coverage_statistics(self, results_dir: str = None) -> Dict[str, Any]:
+        """Load the most recent coverage statistics file"""
+        if results_dir is None:
+            results_dir = Path(__file__).parent / "results"
+        
+        pattern = str(results_dir / "coverage_statistics_*.json")
+        stat_files = glob.glob(pattern)
+        
+        if not stat_files:
+            raise FileNotFoundError(f"No coverage statistics files found in {results_dir}")
+        
+        latest_file = max(stat_files, key=os.path.getmtime)
+        print(f"📊 Loading coverage statistics from: {latest_file}")
+        
+        with open(latest_file, 'r', encoding='utf-8') as f:
+            stats = json.load(f)
+        
+        return stats
+    
+    def generate_coverage_charts(self, stats: Dict[str, Any]) -> str:
+        """Generate coverage analysis charts"""
+        try:
+            fig, axes = plt.subplots(2, 2, figsize=(16, 12))
+            fig.suptitle('OnCall.ai Retrieval Coverage Analysis', fontsize=16, fontweight='bold')
+            
+            category_results = stats['category_results']
+            overall_results = stats['overall_results']
+            
+            # Chart 1: Average Coverage by Category
+            ax1 = axes[0, 0]
+            categories = []
+            avg_coverages = []
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats['successful_evaluations'] > 0:
+                    categories.append(category.replace('_', ' ').title())
+                    avg_coverages.append(cat_stats['average_coverage'] * 100)  # Convert to percentage
+            
+            categories.append('Overall')
+            avg_coverages.append(overall_results['average_coverage'] * 100)
+            
+            bars = ax1.bar(categories, avg_coverages, alpha=0.8, color=['#1f77b4', '#ff7f0e', '#d62728', '#2ca02c'])
+            ax1.set_title('Average Coverage Score by Category', fontweight='bold')
+            ax1.set_ylabel('Coverage Score (%)')
+            ax1.set_xlabel('Query Category')
+            ax1.grid(True, alpha=0.3)
+            
+            # Add target line
+            ax1.axhline(y=40, color='red', linestyle='--', alpha=0.7, label='40% Target')
+            ax1.legend()
+            
+            # Add value labels
+            for bar, coverage in zip(bars, avg_coverages):
+                height = bar.get_height()
+                ax1.text(bar.get_x() + bar.get_width()/2., height + 1,
+                        f'{coverage:.1f}%', ha='center', va='bottom', fontweight='bold')
+            
+            # Chart 2: Coverage Distribution
+            ax2 = axes[0, 1]
+            
+            # Collect all individual coverage scores
+            all_scores = []
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats.get('individual_coverage_scores'):
+                    all_scores.extend([score * 100 for score in cat_stats['individual_coverage_scores']])
+            
+            if all_scores:
+                # Create histogram
+                ax2.hist(all_scores, bins=15, alpha=0.7, color='lightcoral', edgecolor='black')
+                ax2.axvline(x=40, color='red', linestyle='--', alpha=0.7, label='40% Target')
+                ax2.axvline(x=np.mean(all_scores), color='green', linestyle='-', alpha=0.8, label=f'Mean: {np.mean(all_scores):.1f}%')
+                
+                ax2.set_title('Coverage Score Distribution', fontweight='bold')
+                ax2.set_xlabel('Coverage Score (%)')
+                ax2.set_ylabel('Frequency')
+                ax2.legend()
+                ax2.grid(True, alpha=0.3)
+            else:
+                ax2.text(0.5, 0.5, 'No coverage data available', ha='center', va='center', transform=ax2.transAxes)
+                ax2.set_title('Coverage Score Distribution', fontweight='bold')
+            
+            # Chart 3: Statistical Summary Table
+            ax3 = axes[1, 0]
+            ax3.axis('tight')
+            ax3.axis('off')
+            
+            table_data = []
+            headers = ['Category', 'Avg Coverage', 'Min/Max', 'Success/Total', 'Target Met']
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats['total_queries'] > 0:
+                    table_data.append([
+                        category.replace('_', ' ').title(),
+                        f"{cat_stats['average_coverage']:.3f}",
+                        f"{cat_stats['min_coverage']:.3f}/{cat_stats['max_coverage']:.3f}",
+                        f"{cat_stats['successful_evaluations']}/{cat_stats['total_queries']}",
+                        '✅' if cat_stats.get('meets_threshold', False) else '❌'
+                    ])
+            
+            table_data.append([
+                'Overall',
+                f"{overall_results['average_coverage']:.3f}",
+                f"{overall_results['min_coverage']:.3f}/{overall_results['max_coverage']:.3f}",
+                f"{overall_results['successful_queries']}/{overall_results['total_queries']}",
+                '✅' if overall_results.get('meets_threshold', False) else '❌'
+            ])
+            
+            if table_data:
+                table = ax3.table(cellText=table_data, colLabels=headers,
+                                cellLoc='center', loc='center')
+                table.auto_set_font_size(False)
+                table.set_fontsize(10)
+                table.scale(1, 2)
+                
+                # Style header
+                for i in range(len(headers)):
+                    table[(0, i)].set_text_props(weight='bold', color='white')
+                    table[(0, i)].set_facecolor('#2E7D32')
+            
+            ax3.set_title('Coverage Statistics Summary', fontweight='bold', pad=20)
+            
+            # Chart 4: Coverage Performance Radar/Gauge
+            ax4 = axes[1, 1]
+            
+            # Create gauge-like visualization for overall coverage
+            overall_coverage_pct = overall_results['average_coverage'] * 100
+            
+            # Pie chart as gauge
+            sizes = [overall_coverage_pct, 100 - overall_coverage_pct]
+            colors = ['#2ca02c' if overall_coverage_pct >= 40 else '#ff7f0e', '#f0f0f0']
+            
+            wedges, texts, autotexts = ax4.pie(sizes, labels=['Covered', 'Not Covered'],
+                                              autopct='%1.1f%%',
+                                              colors=colors,
+                                              startangle=90,
+                                              counterclock=False)
+            
+            # Add center text
+            ax4.text(0, 0, f'{overall_coverage_pct:.1f}%\nCoverage', 
+                    ha='center', va='center', fontsize=14, fontweight='bold')
+            
+            ax4.set_title(f'Overall Coverage Performance\n{"✅ Target Met" if overall_coverage_pct >= 40 else "❌ Below Target"}', 
+                         fontweight='bold')
+            
+            plt.tight_layout()
+            
+            # Save chart
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            chart_filename = f"coverage_analysis_charts_{timestamp}.png"
+            
+            results_dir = Path(__file__).parent / "results"
+            results_dir.mkdir(exist_ok=True)
+            chart_path = results_dir / chart_filename
+            
+            plt.savefig(chart_path, dpi=300, bbox_inches='tight', facecolor='white')
+            plt.close()
+            
+            print(f"📈 Coverage charts saved to: {chart_path}")
+            return str(chart_path)
+            
+        except Exception as e:
+            print(f"❌ Coverage chart generation failed: {e}")
+            return ""
+
+
+if __name__ == "__main__":
+    """Independent coverage chart generation"""
+    
+    print("📈 OnCall.ai Coverage Chart Generator")
+    
+    chart_gen = CoverageChartGenerator()
+    
+    try:
+        stats = chart_gen.load_latest_coverage_statistics()
+        chart_path = chart_gen.generate_coverage_charts(stats)
+        
+        print(f"\n✅ Coverage chart generation complete!")
+        print(f"📈 Charts saved to: {chart_path}")
+        
+    except FileNotFoundError as e:
+        print(f"❌ {e}")
+        print("💡 Please run latency_evaluator.py first to generate coverage statistics data")
+    except Exception as e:
+        print(f"❌ Chart generation failed: {e}")

evaluation/metric5_6_judge_evaluator_manual.md

@@ -0,0 +1,303 @@
+# Metric 5-6 LLM Judge Evaluator Manual
+
+## Overview
+
+The `metric5_6_llm_judge_evaluator.py` is a multi-system evaluation tool that uses Llama3-70B as a third-party judge to assess medical advice quality across different AI systems. It supports both single-system evaluation and multi-system comparison with a single LLM call for maximum consistency.
+
+## Metrics Evaluated
+
+**Metric 5: Clinical Actionability (臨床可操作性)**
+- Scale: 1-10 (normalized to 0.0-1.0)
+- Question: "Can healthcare providers immediately act on this advice?"
+- Target: ≥7.0/10 for acceptable actionability
+
+**Metric 6: Clinical Evidence Quality (臨床證據品質)**  
+- Scale: 1-10 (normalized to 0.0-1.0)
+- Question: "Is the advice evidence-based and follows medical standards?"
+- Target: ≥7.5/10 for acceptable evidence quality
+
+## System Architecture
+
+### Multi-System Support
+The evaluator supports flexible system combinations:
+- **Single System**: `rag` or `direct`
+- **Two-System Comparison**: `rag,direct` 
+- **Future Extension**: `rag,direct,claude,gpt4` (any combination)
+
+### Judge LLM
+- **Model**: Llama3-70B-Instruct via Hugging Face API
+- **Strategy**: Single batch call for all evaluations
+- **Temperature**: 0.1 (low for consistent evaluation)
+- **Max Tokens**: 2048 (sufficient for evaluation responses)
+
+## Prerequisites
+
+### 1. Environment Setup
+```bash
+# Ensure HF_TOKEN is set in your environment
+export HF_TOKEN="your_huggingface_token"
+
+# Or add to .env file
+echo "HF_TOKEN=your_huggingface_token" >> .env
+```
+
+### 2. Required Data Files
+Before running the judge evaluator, you must have medical outputs from your systems:
+
+**For RAG System**:
+```bash
+python latency_evaluator.py single_test_query.txt
+# Generates: results/medical_outputs_YYYYMMDD_HHMMSS.json
+```
+
+**For Direct LLM System**:
+```bash
+python direct_llm_evaluator.py single_test_query.txt  
+# Generates: results/medical_outputs_direct_YYYYMMDD_HHMMSS.json
+```
+
+## Usage
+
+### Command Line Interface
+
+#### Single System Evaluation
+```bash
+# Evaluate RAG system only
+python metric5_6_llm_judge_evaluator.py rag
+
+# Evaluate Direct LLM system only  
+python metric5_6_llm_judge_evaluator.py direct
+```
+
+#### Multi-System Comparison (Recommended)
+```bash
+# Compare RAG vs Direct systems
+python metric5_6_llm_judge_evaluator.py rag,direct
+
+# Future: Compare multiple systems
+python metric5_6_llm_judge_evaluator.py rag,direct,claude
+```
+
+### Complete Workflow Example
+
+```bash
+# Step 1: Navigate to evaluation directory
+cd /path/to/GenAI-OnCallAssistant/evaluation
+
+# Step 2: Generate medical outputs from both systems
+python latency_evaluator.py single_test_query.txt
+python direct_llm_evaluator.py single_test_query.txt
+
+# Step 3: Run comparative evaluation
+python metric5_6_llm_judge_evaluator.py rag,direct
+```
+
+## Output Files
+
+### Generated Files
+- **Statistics**: `results/judge_evaluation_comparison_rag_vs_direct_YYYYMMDD_HHMMSS.json`
+- **Detailed Results**: Stored in evaluator's internal results array
+
+### File Structure
+```json
+{
+  "comparison_metadata": {
+    "systems_compared": ["rag", "direct"],
+    "comparison_type": "multi_system",
+    "timestamp": "2025-08-04T22:00:00"
+  },
+  "category_results": {
+    "diagnosis": {
+      "average_actionability": 0.850,
+      "average_evidence": 0.780,
+      "query_count": 1,
+      "actionability_target_met": true,
+      "evidence_target_met": true
+    }
+  },
+  "overall_results": {
+    "average_actionability": 0.850,
+    "average_evidence": 0.780,
+    "successful_evaluations": 2,
+    "total_queries": 2,
+    "actionability_target_met": true,
+    "evidence_target_met": true
+  }
+}
+```
+
+## Evaluation Process
+
+### 1. File Discovery
+The evaluator automatically finds the latest medical output files:
+- **RAG**: `medical_outputs_*.json`
+- **Direct**: `medical_outputs_direct_*.json`
+- **Custom**: `medical_outputs_{system}_*.json`
+
+### 2. Prompt Generation
+For multi-system comparison, the evaluator creates a structured prompt:
+```
+You are a medical expert evaluating and comparing AI systems...
+
+SYSTEM 1 (RAG): Uses medical guidelines + LLM for evidence-based advice
+SYSTEM 2 (Direct): Uses LLM only without external guidelines
+
+QUERY 1 (DIAGNOSIS):
+Patient Query: 60-year-old patient with hypertension history...
+
+SYSTEM 1 Response: For a 60-year-old patient with...
+SYSTEM 2 Response: Based on the symptoms described...
+
+RESPONSE FORMAT:
+Query 1 System 1: Actionability=X, Evidence=Y
+Query 1 System 2: Actionability=X, Evidence=Y
+```
+
+### 3. LLM Judge Evaluation
+- **Single API Call**: All systems evaluated in one request for consistency
+- **Response Parsing**: Automatic extraction of numerical scores
+- **Error Handling**: Graceful handling of parsing failures
+
+### 4. Results Analysis
+- **System-Specific Statistics**: Individual performance metrics
+- **Comparative Analysis**: Direct system-to-system comparison
+- **Target Compliance**: Automatic threshold checking
+
+## Expected Output
+
+### Console Output Example
+```
+🧠 OnCall.ai LLM Judge Evaluator - Metrics 5-6 Multi-System Evaluation
+
+🧪 Multi-System Comparison: RAG vs DIRECT
+📊 Found rag outputs: results/medical_outputs_20250804_215917.json
+📊 Found direct outputs: results/medical_outputs_direct_20250804_220000.json
+📊 Comparing 2 systems with 1 queries each
+🎯 Metrics: 5 (Actionability) + 6 (Evidence Quality)
+⚡ Strategy: Single comparison call for maximum consistency
+
+🧠 Multi-system comparison: rag, direct
+📊 Evaluating 1 queries across 2 systems...
+📝 Comparison prompt created (2150 characters)
+🔄 Calling judge LLM for multi-system comparison...
+✅ Judge LLM completed comparison evaluation in 45.3s
+📄 Response length: 145 characters
+📊 RAG: 1 evaluations parsed
+📊 DIRECT: 1 evaluations parsed
+
+📊 === LLM JUDGE EVALUATION SUMMARY ===
+Systems Compared: RAG vs DIRECT
+Overall Performance:
+   Average Actionability: 0.850 (8.5/10)
+   Average Evidence Quality: 0.780 (7.8/10)
+   Actionability Target (≥7.0): ✅ Met
+   Evidence Target (≥7.5): ✅ Met
+
+System Breakdown:
+   RAG: Actionability=0.900, Evidence=0.850 [1 queries]
+   DIRECT: Actionability=0.800, Evidence=0.710 [1 queries]
+
+✅ LLM judge evaluation complete!
+📊 Statistics: results/judge_evaluation_comparison_rag_vs_direct_20250804_220000.json
+⚡ Efficiency: 2 evaluations in 1 LLM call
+```
+
+## Key Features
+
+### 1. Scientific Comparison Design
+- **Single Judge Call**: All systems evaluated simultaneously for consistency
+- **Eliminates Temporal Bias**: Same judge, same context, same standards
+- **Direct System Comparison**: Side-by-side evaluation format
+
+### 2. Flexible Architecture  
+- **Backward Compatible**: Single system evaluation still supported
+- **Future Extensible**: Easy to add new systems (`claude`, `gpt4`, etc.)
+- **Modular Design**: Clean separation of concerns
+
+### 3. Robust Error Handling
+- **File Validation**: Automatic detection of missing input files
+- **Query Count Verification**: Warns if systems have different query counts
+- **Graceful Degradation**: Continues operation despite partial failures
+
+### 4. Comprehensive Reporting
+- **System-Specific Metrics**: Individual performance analysis
+- **Comparative Statistics**: Direct system-to-system comparison
+- **Target Compliance**: Automatic benchmark checking
+- **Detailed Metadata**: Full traceability of evaluation parameters
+
+## Troubleshooting
+
+### Common Issues
+
+#### 1. Missing Input Files
+```
+❌ No medical outputs files found for rag system
+💡 Please run evaluators first:
+   python latency_evaluator.py single_test_query.txt
+```
+**Solution**: Run the prerequisite evaluators to generate medical outputs.
+
+#### 2. HF_TOKEN Not Set
+```
+❌ HF_TOKEN is missing from environment variables
+```
+**Solution**: Set your Hugging Face token in environment or `.env` file.
+
+#### 3. Query Count Mismatch
+```
+⚠️ Warning: Systems have different query counts: {'rag': 3, 'direct': 1}
+```
+**Solution**: Ensure both systems processed the same input file.
+
+#### 4. LLM API Timeout
+```
+❌ Multi-system evaluation failed: timeout
+```
+**Solution**: Check internet connection and Hugging Face API status.
+
+### Debug Tips
+
+1. **Check File Existence**: Verify medical output files in `results/` directory
+2. **Validate JSON Format**: Ensure input files are properly formatted
+3. **Monitor API Usage**: Check Hugging Face account limits
+4. **Review Logs**: Examine detailed logging output for specific errors
+
+## Future Extensions
+
+### Phase 2: Generic Multi-System Framework
+```bash
+# Configuration-driven system comparison
+python metric5_6_llm_judge_evaluator.py --config comparison_config.json
+```
+
+### Phase 3: Unlimited System Support
+```bash
+# Dynamic system registration
+python metric5_6_llm_judge_evaluator.py med42,claude,gpt4,palm,llama2
+```
+
+### Integration with Chart Generators
+```bash
+# Generate comparison visualizations
+python metric5_6_llm_judge_chart_generator.py rag,direct
+```
+
+## Best Practices
+
+1. **Consistent Test Data**: Use the same query file for all systems
+2. **Sequential Execution**: Complete data collection before evaluation
+3. **Batch Processing**: Use multi-system mode for scientific comparison
+4. **Result Verification**: Review detailed statistics files for accuracy
+5. **Performance Monitoring**: Track evaluation latency and API costs
+
+## Scientific Validity
+
+The multi-system comparison approach provides superior scientific validity compared to separate evaluations:
+
+- **Eliminates Judge Variability**: Same judge evaluates all systems
+- **Reduces Temporal Effects**: All evaluations in single time window  
+- **Ensures Consistent Standards**: Identical evaluation criteria applied
+- **Enables Direct Comparison**: Side-by-side system assessment
+- **Maximizes Efficiency**: Single API call vs multiple separate calls
+
+This design makes the evaluation results more reliable for research publications and system optimization decisions.

evaluation/metric5_6_llm_judge_chart_generator.py

@@ -0,0 +1,430 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - LLM Judge Chart Generator (Metrics 5-6)
+==========================================================
+
+Generates comprehensive comparison charts for LLM judge evaluation results.
+Supports both single-system and multi-system visualization with professional layouts.
+
+Metrics visualized:
+5. Clinical Actionability (臨床可操作性) - 1-10 scale
+6. Clinical Evidence Quality (臨床證據品質) - 1-10 scale
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any, Tuple
+from datetime import datetime
+from pathlib import Path
+import glob
+import numpy as np
+
+# Visualization imports
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+from matplotlib.patches import Rectangle
+
+
+class LLMJudgeChartGenerator:
+    """Generate professional comparison charts for LLM judge evaluation results"""
+    
+    def __init__(self):
+        """Initialize chart generator with professional styling"""
+        print("📈 Initializing LLM Judge Chart Generator...")
+        
+        # Set up professional chart style
+        plt.style.use('default')
+        sns.set_palette("husl")
+        
+        # Professional color scheme for medical evaluation
+        self.colors = {
+            'rag': '#2E8B57',      # Sea Green - represents evidence-based
+            'direct': '#4682B4',   # Steel Blue - represents direct approach
+            'claude': '#9370DB',   # Medium Purple - future extension
+            'gpt4': '#DC143C',     # Crimson - future extension
+            'actionability': '#FF6B6B',  # Coral Red
+            'evidence': '#4ECDC4',        # Turquoise
+            'target_line': '#FF4444',     # Red for target thresholds
+            'grid': '#E0E0E0'             # Light gray for grid
+        }
+        
+        print("✅ Chart Generator ready with professional medical styling")
+    
+    def load_latest_statistics(self, results_dir: str = None) -> Dict[str, Any]:
+        """
+        Load the most recent judge evaluation statistics file
+        
+        Args:
+            results_dir: Directory containing statistics files
+        """
+        if results_dir is None:
+            results_dir = Path(__file__).parent / "results"
+        
+        # Find latest comparison statistics file
+        pattern = str(results_dir / "judge_evaluation_comparison_*.json")
+        stat_files = glob.glob(pattern)
+        
+        if not stat_files:
+            raise FileNotFoundError(f"No judge evaluation comparison files found in {results_dir}")
+        
+        # Get the most recent file
+        latest_file = max(stat_files, key=os.path.getmtime)
+        
+        print(f"📊 Loading statistics from: {latest_file}")
+        
+        with open(latest_file, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    
+    def generate_comparison_charts(self, stats: Dict[str, Any], save_path: str = None) -> str:
+        """
+        Generate comprehensive 4-panel comparison visualization
+        
+        Creates professional charts showing:
+        1. System comparison radar chart
+        2. Grouped bar chart comparison
+        3. Actionability vs Evidence scatter plot
+        4. Category-wise heatmap
+        """
+        try:
+            # Create figure with subplots
+            fig, axes = plt.subplots(2, 2, figsize=(16, 12))
+            fig.suptitle(
+                'Medical AI Systems Comparison - Clinical Quality Assessment\n'
+                'Actionability (1-10): Can healthcare providers act immediately? | '
+                'Evidence Quality (1-10): Is advice evidence-based?',
+                fontsize=14, fontweight='bold', y=0.95
+            )
+            
+            # Extract comparison metadata
+            comparison_meta = stats.get('comparison_metadata', {})
+            systems = comparison_meta.get('systems_compared', ['rag', 'direct'])
+            
+            overall_results = stats['overall_results']
+            category_results = stats['category_results']
+            
+            # Chart 1: System Comparison Radar Chart
+            self._create_radar_chart(axes[0, 0], stats, systems)
+            
+            # Chart 2: Grouped Bar Chart Comparison
+            self._create_grouped_bar_chart(axes[0, 1], stats, systems)
+            
+            # Chart 3: Actionability vs Evidence Scatter Plot
+            self._create_scatter_plot(axes[1, 0], stats, systems)
+            
+            # Chart 4: Category-wise Performance Heatmap
+            self._create_heatmap(axes[1, 1], stats, systems)
+            
+            # Add method annotation at bottom
+            method_text = (
+                f"Evaluation: Llama3-70B judge | Targets: Actionability ≥7.0, Evidence ≥7.5 | "
+                f"Systems: {', '.join([s.upper() for s in systems])} | "
+                f"Queries: {overall_results.get('total_queries', 'N/A')}"
+            )
+            fig.text(0.5, 0.02, method_text, ha='center', fontsize=10, 
+                    style='italic', color='gray')
+            
+            # Adjust layout
+            plt.tight_layout()
+            plt.subplots_adjust(top=0.88, bottom=0.08)
+            
+            # Save the chart
+            if save_path is None:
+                timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+                systems_str = "_vs_".join(systems)
+                save_path = f"judge_comparison_charts_{systems_str}_{timestamp}.png"
+            
+            results_dir = Path(__file__).parent / "results"
+            results_dir.mkdir(exist_ok=True)
+            full_path = results_dir / save_path
+            
+            plt.savefig(full_path, dpi=300, bbox_inches='tight')
+            plt.show()
+            
+            print(f"📊 Comparison charts saved to: {full_path}")
+            return str(full_path)
+            
+        except Exception as e:
+            print(f"❌ Chart generation failed: {e}")
+            raise
+    
+    def _create_radar_chart(self, ax, stats: Dict, systems: List[str]):
+        """Create radar chart for multi-dimensional system comparison"""
+        ax.set_title('Multi-Dimensional System Comparison', fontweight='bold', pad=20)
+        
+        # Prepare data for radar chart using real system-specific data
+        categories = ['Overall Actionability', 'Overall Evidence', 'Diagnosis', 'Treatment', 'Mixed']
+        
+        # Extract real system-specific metrics
+        detailed_results = stats.get('detailed_system_results', {})
+        system_data = {}
+        
+        for system in systems:
+            if system in detailed_results:
+                system_info = detailed_results[system]
+                system_results = system_info['results']
+                
+                # Calculate category-specific performance
+                category_performance = {}
+                for result in system_results:
+                    category = result.get('category', 'unknown').lower()
+                    if category not in category_performance:
+                        category_performance[category] = {'actionability': [], 'evidence': []}
+                    category_performance[category]['actionability'].append(result['actionability_score'])
+                    category_performance[category]['evidence'].append(result['evidence_score'])
+                
+                # Build radar chart data
+                system_scores = [
+                    system_info['avg_actionability'],  # Overall Actionability
+                    system_info['avg_evidence'],       # Overall Evidence
+                    # Category-specific scores (average of actionability and evidence)
+                    (sum(category_performance.get('diagnosis', {}).get('actionability', [0])) / 
+                     len(category_performance.get('diagnosis', {}).get('actionability', [1])) + 
+                     sum(category_performance.get('diagnosis', {}).get('evidence', [0])) / 
+                     len(category_performance.get('diagnosis', {}).get('evidence', [1]))) / 2 if 'diagnosis' in category_performance else 0.5,
+                    
+                    (sum(category_performance.get('treatment', {}).get('actionability', [0])) / 
+                     len(category_performance.get('treatment', {}).get('actionability', [1])) + 
+                     sum(category_performance.get('treatment', {}).get('evidence', [0])) / 
+                     len(category_performance.get('treatment', {}).get('evidence', [1]))) / 2 if 'treatment' in category_performance else 0.5,
+                    
+                    (sum(category_performance.get('mixed', {}).get('actionability', [0])) / 
+                     len(category_performance.get('mixed', {}).get('actionability', [1])) + 
+                     sum(category_performance.get('mixed', {}).get('evidence', [0])) / 
+                     len(category_performance.get('mixed', {}).get('evidence', [1]))) / 2 if 'mixed' in category_performance else 0.5
+                ]
+                system_data[system] = system_scores
+            else:
+                # Fallback to overall stats if detailed results not available
+                overall_results = stats['overall_results']
+                system_data[system] = [
+                    overall_results['average_actionability'],
+                    overall_results['average_evidence'],
+                    0.7, 0.6, 0.5  # Placeholder for missing category data
+                ]
+        
+        # Create radar chart
+        angles = np.linspace(0, 2 * np.pi, len(categories), endpoint=False).tolist()
+        angles += angles[:1]  # Complete the circle
+        
+        for system in systems:
+            values = system_data[system] + [system_data[system][0]]  # Complete the circle
+            ax.plot(angles, values, 'o-', linewidth=2, 
+                   label=f'{system.upper()} System', color=self.colors.get(system, 'gray'))
+            ax.fill(angles, values, alpha=0.1, color=self.colors.get(system, 'gray'))
+        
+        # Customize radar chart
+        ax.set_xticks(angles[:-1])
+        ax.set_xticklabels(categories, fontsize=9)
+        ax.set_ylim(0, 1)
+        ax.set_yticks([0.2, 0.4, 0.6, 0.8, 1.0])
+        ax.set_yticklabels(['2.0', '4.0', '6.0', '8.0', '10.0'])
+        ax.grid(True, alpha=0.3)
+        ax.legend(loc='upper right', bbox_to_anchor=(1.2, 1.0))
+        
+        # Add target threshold circle
+        target_circle = [0.7] * (len(categories) + 1)  # 7.0 threshold
+        ax.plot(angles, target_circle, '--', color=self.colors['target_line'], 
+               alpha=0.7, label='Target (7.0)')
+    
+    def _create_grouped_bar_chart(self, ax, stats: Dict, systems: List[str]):
+        """Create grouped bar chart for direct metric comparison"""
+        ax.set_title('Direct Metric Comparison', fontweight='bold', pad=20)
+        
+        # Prepare data using real system-specific metrics
+        metrics = ['Actionability', 'Evidence Quality']
+        detailed_results = stats.get('detailed_system_results', {})
+        
+        # Extract real system-specific data
+        system_scores = {}
+        for system in systems:
+            if system in detailed_results:
+                system_info = detailed_results[system]
+                system_scores[system] = [
+                    system_info['avg_actionability'],
+                    system_info['avg_evidence']
+                ]
+            else:
+                # Fallback to overall results
+                overall_results = stats['overall_results']
+                system_scores[system] = [
+                    overall_results['average_actionability'],
+                    overall_results['average_evidence']
+                ]
+        
+        # Create grouped bar chart
+        x = np.arange(len(metrics))
+        width = 0.35 if len(systems) == 2 else 0.25
+        
+        for i, system in enumerate(systems):
+            offset = (i - len(systems)/2 + 0.5) * width
+            bars = ax.bar(x + offset, system_scores[system], width,
+                         label=f'{system.upper()}', color=self.colors.get(system, 'gray'),
+                         alpha=0.8)
+            
+            # Add value labels on bars
+            for bar, value in zip(bars, system_scores[system]):
+                height = bar.get_height()
+                ax.text(bar.get_x() + bar.get_width()/2., height + 0.01,
+                       f'{value:.3f}', ha='center', va='bottom', fontweight='bold')
+        
+        # Add target threshold lines
+        ax.axhline(y=0.7, color=self.colors['target_line'], linestyle='--', 
+                  alpha=0.7, label='Actionability Target (7.0)')
+        ax.axhline(y=0.75, color=self.colors['target_line'], linestyle=':', 
+                  alpha=0.7, label='Evidence Target (7.5)')
+        
+        # Customize chart
+        ax.set_xlabel('Evaluation Metrics')
+        ax.set_ylabel('Score (0-1 scale)')
+        ax.set_title('System Performance Comparison')
+        ax.set_xticks(x)
+        ax.set_xticklabels(metrics)
+        ax.legend(loc='upper left')
+        ax.grid(True, alpha=0.3, axis='y')
+        ax.set_ylim(0, 1.0)
+    
+    def _create_scatter_plot(self, ax, stats: Dict, systems: List[str]):
+        """Create scatter plot for actionability vs evidence quality analysis"""
+        ax.set_title('Actionability vs Evidence Quality Analysis', fontweight='bold', pad=20)
+        
+        # Extract real query-level data from detailed results
+        detailed_results = stats.get('detailed_system_results', {})
+        
+        for system in systems:
+            if system in detailed_results:
+                system_results = detailed_results[system]['results']
+                
+                # Extract real actionability and evidence scores for each query
+                actionability_scores = [r['actionability_score'] for r in system_results]
+                evidence_scores = [r['evidence_score'] for r in system_results]
+                
+                ax.scatter(actionability_scores, evidence_scores, 
+                          label=f'{system.upper()}', color=self.colors.get(system, 'gray'),
+                          alpha=0.7, s=100, edgecolors='white', linewidth=1)
+            else:
+                # Fallback: create single point from overall averages
+                overall_results = stats['overall_results']
+                ax.scatter([overall_results['average_actionability']], 
+                          [overall_results['average_evidence']], 
+                          label=f'{system.upper()}', color=self.colors.get(system, 'gray'),
+                          alpha=0.7, s=100, edgecolors='white', linewidth=1)
+        
+        # Add target threshold lines
+        ax.axvline(x=0.7, color=self.colors['target_line'], linestyle='--', 
+                  alpha=0.7, label='Actionability Target')
+        ax.axhline(y=0.75, color=self.colors['target_line'], linestyle='--', 
+                  alpha=0.7, label='Evidence Target')
+        
+        # Add target zone
+        target_rect = Rectangle((0.7, 0.75), 0.3, 0.25, linewidth=1, 
+                               edgecolor=self.colors['target_line'], facecolor='green', 
+                               alpha=0.1, label='Target Zone')
+        ax.add_patch(target_rect)
+        
+        # Customize chart
+        ax.set_xlabel('Clinical Actionability (0-1 scale)')
+        ax.set_ylabel('Clinical Evidence Quality (0-1 scale)')
+        ax.legend(loc='lower right')
+        ax.grid(True, alpha=0.3)
+        ax.set_xlim(0, 1)
+        ax.set_ylim(0, 1)
+    
+    def _create_heatmap(self, ax, stats: Dict, systems: List[str]):
+        """Create heatmap for category-wise performance matrix"""
+        ax.set_title('Category-wise Performance Matrix', fontweight='bold', pad=20)
+        
+        # Prepare data
+        categories = ['Diagnosis', 'Treatment', 'Mixed']
+        metrics = ['Actionability', 'Evidence']
+        category_results = stats['category_results']
+        
+        # Create data matrix
+        data_matrix = []
+        row_labels = []
+        
+        for system in systems:
+            for metric in metrics:
+                row_data = []
+                for category in categories:
+                    cat_key = category.lower()
+                    if cat_key in category_results and category_results[cat_key]['query_count'] > 0:
+                        if metric == 'Actionability':
+                            value = category_results[cat_key]['average_actionability']
+                        else:
+                            value = category_results[cat_key]['average_evidence']
+                    else:
+                        value = 0.5  # Placeholder for missing data
+                    row_data.append(value)
+                
+                data_matrix.append(row_data)
+                row_labels.append(f'{system.upper()}\n{metric}')
+        
+        # Create heatmap
+        im = ax.imshow(data_matrix, cmap='RdYlGn', aspect='auto', vmin=0, vmax=1)
+        
+        # Set ticks and labels
+        ax.set_xticks(np.arange(len(categories)))
+        ax.set_yticks(np.arange(len(row_labels)))
+        ax.set_xticklabels(categories)
+        ax.set_yticklabels(row_labels, fontsize=9)
+        
+        # Add text annotations
+        for i in range(len(row_labels)):
+            for j in range(len(categories)):
+                text = ax.text(j, i, f'{data_matrix[i][j]:.3f}',
+                             ha='center', va='center', fontweight='bold',
+                             color='white' if data_matrix[i][j] < 0.5 else 'black')
+        
+        # Add colorbar
+        cbar = plt.colorbar(im, ax=ax, shrink=0.6)
+        cbar.set_label('Performance Score (0-1)', rotation=270, labelpad=15)
+        
+        ax.set_xlabel('Query Categories')
+        ax.set_ylabel('System × Metric')
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent chart generation interface"""
+    
+    print("📊 OnCall.ai LLM Judge Chart Generator - Metrics 5-6 Visualization")
+    
+    # Initialize generator
+    generator = LLMJudgeChartGenerator()
+    
+    try:
+        # Load latest statistics
+        stats = generator.load_latest_statistics()
+        
+        print(f"📈 Generating comparison charts...")
+        
+        # Generate comprehensive comparison charts
+        chart_path = generator.generate_comparison_charts(stats)
+        
+        # Print summary
+        comparison_meta = stats.get('comparison_metadata', {})
+        systems = comparison_meta.get('systems_compared', ['rag', 'direct'])
+        overall_results = stats['overall_results']
+        
+        print(f"\n📊 === CHART GENERATION SUMMARY ===")
+        print(f"Systems Visualized: {' vs '.join([s.upper() for s in systems])}")
+        print(f"Overall Actionability: {overall_results['average_actionability']:.3f}")
+        print(f"Overall Evidence Quality: {overall_results['average_evidence']:.3f}")
+        print(f"Total Queries: {overall_results['total_queries']}")
+        print(f"Chart Components: Radar Chart, Bar Chart, Scatter Plot, Heatmap")
+        
+        print(f"\n✅ Comprehensive visualization complete!")
+        print(f"📊 Charts saved to: {chart_path}")
+        print(f"💡 Tip: Charts optimized for research presentations and publications")
+        
+    except FileNotFoundError as e:
+        print(f"❌ {e}")
+        print(f"💡 Please run judge evaluation first:")
+        print("   python metric5_6_llm_judge_evaluator.py rag,direct")
+    except Exception as e:
+        print(f"❌ Chart generation failed: {e}")

evaluation/metric5_6_llm_judge_evaluator.py

@@ -0,0 +1,643 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - LLM Judge Evaluator (Metrics 5-6)
+====================================================
+
+Uses Llama3-70B as third-party judge to evaluate medical advice quality.
+Batch evaluation strategy: 1 call evaluates all queries for maximum efficiency.
+
+Metrics evaluated:
+5. Clinical Actionability (臨床可操作性)
+6. Clinical Evidence Quality (臨床證據品質)
+
+EVALUATION RUBRICS:
+
+Metric 5: Clinical Actionability (1-10 scale)
+  1-2 points: Almost no actionable advice; extremely abstract or empty responses.
+  3-4 points: Provides some directional suggestions but too vague, lacks clear steps.
+  5-6 points: Offers basic executable steps but lacks details or insufficient explanation for key aspects.
+  7-8 points: Clear and complete steps that clinicians can follow, with occasional gaps needing supplementation.
+  9-10 points: Extremely actionable with precise, step-by-step executable guidance; can be used "as-is" immediately.
+
+Metric 6: Clinical Evidence Quality (1-10 scale)
+  1-2 points: Almost no evidence support; cites completely irrelevant or unreliable sources.
+  3-4 points: References lower quality literature or guidelines, or sources lack authority.
+  5-6 points: Uses general quality literature/guidelines but lacks depth or currency.
+  7-8 points: References reliable, authoritative sources (renowned journals or authoritative guidelines) with accurate explanations.
+  9-10 points: Rich and high-quality evidence sources (systematic reviews, RCTs, etc.) combined with latest research; enhances recommendation credibility.
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+import time
+from typing import Dict, List, Any, Tuple
+from datetime import datetime
+from pathlib import Path
+import glob
+import re
+
+# Evaluation Rubrics as programmable constants
+ACTIONABILITY_RUBRIC = {
+    (1, 2): "Almost no actionable advice; extremely abstract or empty responses.",
+    (3, 4): "Provides some directional suggestions but too vague, lacks clear steps.",
+    (5, 6): "Offers basic executable steps but lacks details or insufficient explanation for key aspects.",
+    (7, 8): "Clear and complete steps that clinicians can follow, with occasional gaps needing supplementation.",
+    (9, 10): "Extremely actionable with precise, step-by-step executable guidance; can be used 'as-is' immediately."
+}
+
+EVIDENCE_RUBRIC = {
+    (1, 2): "Almost no evidence support; cites completely irrelevant or unreliable sources.",
+    (3, 4): "References lower quality literature or guidelines, or sources lack authority.",
+    (5, 6): "Uses general quality literature/guidelines but lacks depth or currency.",
+    (7, 8): "References reliable, authoritative sources (renowned journals or authoritative guidelines) with accurate explanations.",
+    (9, 10): "Rich and high-quality evidence sources (systematic reviews, RCTs, etc.) combined with latest research; enhances recommendation credibility."
+}
+
+def print_evaluation_rubrics():
+    """Print detailed evaluation rubrics for reference"""
+    print("=" * 60)
+    print("CLINICAL EVALUATION RUBRICS")
+    print("=" * 60)
+    
+    print("\n🎯 METRIC 5: Clinical Actionability (1-10 scale)")
+    print("-" * 50)
+    for score_range, description in ACTIONABILITY_RUBRIC.items():
+        print(f"{score_range[0]}–{score_range[1]} points: {description}")
+    
+    print("\n📚 METRIC 6: Clinical Evidence Quality (1-10 scale)")
+    print("-" * 50)
+    for score_range, description in EVIDENCE_RUBRIC.items():
+        print(f"{score_range[0]}–{score_range[1]} points: {description}")
+    
+    print("\n" + "=" * 60)
+    print("TARGET THRESHOLDS:")
+    print("• Actionability: ≥7.0 (Acceptable clinical utility)")
+    print("• Evidence Quality: ≥7.5 (Reliable evidence support)")
+    print("=" * 60)
+
+def get_rubric_description(score: int, metric_type: str) -> str:
+    """Get rubric description for a given score and metric type"""
+    rubric = ACTIONABILITY_RUBRIC if metric_type == "actionability" else EVIDENCE_RUBRIC
+    
+    for score_range, description in rubric.items():
+        if score_range[0] <= score <= score_range[1]:
+            return description
+    
+    return "Score out of valid range (1-10)"
+
+# Add project path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import LLM client for judge evaluation
+try:
+    from llm_clients import llm_Llama3_70B_JudgeClient
+except ImportError as e:
+    print(f"❌ Import failed: {e}")
+    print("Please ensure running from project root directory")
+    sys.exit(1)
+
+
+class LLMJudgeEvaluator:
+    """LLM judge evaluator using batch evaluation strategy"""
+    
+    def __init__(self):
+        """Initialize judge LLM client"""
+        print("🔧 Initializing LLM Judge Evaluator...")
+        
+        # Initialize Llama3-70B as judge LLM
+        self.judge_llm = llm_Llama3_70B_JudgeClient()
+        
+        self.evaluation_results = []
+        
+        print("✅ LLM Judge Evaluator initialization complete")
+    
+    def load_medical_outputs(self, filepath: str) -> List[Dict[str, Any]]:
+        """Load medical outputs from file"""
+        print(f"📁 Loading medical outputs from: {filepath}")
+        
+        with open(filepath, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        
+        medical_outputs = data.get('medical_outputs', [])
+        print(f"📋 Loaded {len(medical_outputs)} medical outputs")
+        
+        return medical_outputs
+    
+    def find_medical_outputs_for_systems(self, systems: List[str]) -> Dict[str, str]:
+        """Find medical outputs files for multiple systems"""
+        results_dir = Path(__file__).parent / "results"
+        system_files = {}
+        
+        for system in systems:
+            if system == "rag":
+                # Use more specific pattern to exclude direct files
+                pattern = str(results_dir / "medical_outputs_[0-9]*.json")
+            elif system == "direct":
+                pattern = str(results_dir / "medical_outputs_direct_*.json")
+            else:
+                # Future extension: support other systems
+                pattern = str(results_dir / f"medical_outputs_{system}_*.json")
+            
+            print(f"🔍 Searching for {system} with pattern: {pattern}")
+            output_files = glob.glob(pattern)
+            print(f"🔍 Found files for {system}: {output_files}")
+            
+            if not output_files:
+                raise FileNotFoundError(f"No medical outputs files found for {system} system")
+            
+            latest_file = max(output_files, key=os.path.getmtime)
+            system_files[system] = latest_file
+            print(f"📊 Found {system} outputs: {latest_file}")
+        
+        return system_files
+    
+    def create_comparison_evaluation_prompt(self, systems_outputs: Dict[str, List[Dict]]) -> str:
+        """
+        Create comparison evaluation prompt for multiple systems
+        
+        Args:
+            systems_outputs: Dict mapping system names to their medical outputs
+        """
+        system_names = list(systems_outputs.keys())
+        
+        prompt_parts = [
+            "You are a medical expert evaluating and comparing AI systems for clinical advice quality.",
+            f"Please evaluate {len(system_names)} different systems using the detailed rubrics below:",
+            "",
+            "EVALUATION RUBRICS:",
+            "",
+            "METRIC 1: Clinical Actionability (1-10 scale)",
+            "Question: Can healthcare providers immediately act on this advice?",
+            "1-2 points: Almost no actionable advice; extremely abstract or empty responses.",
+            "3-4 points: Provides directional suggestions but too vague, lacks clear steps.",
+            "5-6 points: Offers basic executable steps but lacks details for key aspects.",
+            "7-8 points: Clear and complete steps that clinicians can follow with occasional gaps.",
+            "9-10 points: Extremely actionable with precise, step-by-step executable guidance.",
+            "",
+            "METRIC 2: Clinical Evidence Quality (1-10 scale)", 
+            "Question: Is the advice evidence-based and follows medical standards?",
+            "1-2 points: Almost no evidence support; cites irrelevant or unreliable sources.",
+            "3-4 points: References lower quality literature or sources lack authority.",
+            "5-6 points: Uses general quality literature/guidelines but lacks depth or currency.",
+            "7-8 points: References reliable, authoritative sources with accurate explanations.",
+            "9-10 points: Rich, high-quality evidence sources combined with latest research.",
+            "",
+            "TARGET THRESHOLDS: Actionability ≥7.0, Evidence Quality ≥7.5",
+            ""
+        ]
+        
+        # Add system descriptions
+        for i, system in enumerate(system_names, 1):
+            if system == "rag":
+                prompt_parts.append(f"SYSTEM {i} (RAG): Uses medical guidelines + LLM for evidence-based advice")
+            elif system == "direct":
+                prompt_parts.append(f"SYSTEM {i} (Direct): Uses LLM only without external guidelines")
+            else:
+                prompt_parts.append(f"SYSTEM {i} ({system.upper()}): {system} medical AI system")
+        
+        prompt_parts.extend([
+            "",
+            "EVALUATION CRITERIA:",
+            "1. Clinical Actionability (1-10): Can healthcare providers immediately act on this advice?",
+            "2. Clinical Evidence Quality (1-10): Is the advice evidence-based and follows medical standards?",
+            "",
+            "QUERIES TO EVALUATE:",
+            ""
+        ])
+        
+        # Get all queries (assuming all systems processed same queries)
+        first_system = system_names[0]
+        queries = systems_outputs[first_system]
+        
+        # Add each query with all system responses
+        for i, query_data in enumerate(queries, 1):
+            query = query_data.get('query', '')
+            category = query_data.get('category', 'unknown')
+            
+            prompt_parts.extend([
+                f"=== QUERY {i} ({category.upper()}) ===",
+                f"Patient Query: {query}",
+                ""
+            ])
+            
+            # Add each system's response
+            for j, system in enumerate(system_names, 1):
+                system_query = systems_outputs[system][i-1]  # Get corresponding query from this system
+                advice = system_query.get('medical_advice', '')
+                
+                prompt_parts.extend([
+                    f"SYSTEM {j} Response: {advice}",
+                    ""
+                ])
+        
+        prompt_parts.extend([
+            "RESPONSE FORMAT (provide exactly this format):",
+            ""
+        ])
+        
+        # Add response format template
+        for i in range(1, len(queries) + 1):
+            for j, system in enumerate(system_names, 1):
+                prompt_parts.append(f"Query {i} System {j}: Actionability=X, Evidence=Y")
+        
+        prompt_parts.extend([
+            "",
+            "Replace X and Y with numeric scores 1-10.",
+            "Provide only the scores in the exact format above.",
+            f"Note: System 1={system_names[0]}, System 2={system_names[1] if len(system_names) > 1 else 'N/A'}"
+        ])
+        
+        return "\n".join(prompt_parts)
+    
+    def parse_comparison_evaluation_response(self, response: str, systems_outputs: Dict[str, List[Dict]]) -> Dict[str, List[Dict]]:
+        """Parse comparison evaluation response into results by system"""
+        results_by_system = {}
+        system_names = list(systems_outputs.keys())
+        
+        # Initialize results for each system
+        for system in system_names:
+            results_by_system[system] = []
+        
+        lines = response.strip().split('\n')
+        
+        for line in lines:
+            line = line.strip()
+            if not line:
+                continue
+            
+            # Parse format: "Query X System Y: Actionability=A, Evidence=B"
+            match = re.match(r'Query\s+(\d+)\s+System\s+(\d+):\s*Actionability\s*=\s*(\d+)\s*,\s*Evidence\s*=\s*(\d+)', line, re.IGNORECASE)
+            
+            if match:
+                query_num = int(match.group(1)) - 1  # 0-based index
+                system_num = int(match.group(2)) - 1  # 0-based index
+                actionability_score = int(match.group(3))
+                evidence_score = int(match.group(4))
+                
+                if system_num < len(system_names) and query_num < len(systems_outputs[system_names[system_num]]):
+                    system_name = system_names[system_num]
+                    output = systems_outputs[system_name][query_num]
+                    
+                    result = {
+                        "query": output.get('query', ''),
+                        "category": output.get('category', 'unknown'),
+                        "system_type": system_name,
+                        "medical_advice": output.get('medical_advice', ''),
+                        
+                        # Metric 5: Clinical Actionability
+                        "actionability_score": actionability_score / 10.0,
+                        "actionability_raw": actionability_score,
+                        
+                        # Metric 6: Clinical Evidence Quality
+                        "evidence_score": evidence_score / 10.0,
+                        "evidence_raw": evidence_score,
+                        
+                        "evaluation_success": True,
+                        "timestamp": datetime.now().isoformat()
+                    }
+                    
+                    results_by_system[system_name].append(result)
+        
+        return results_by_system
+    
+    def evaluate_multiple_systems(self, systems_outputs: Dict[str, List[Dict]]) -> Dict[str, List[Dict]]:
+        """
+        Evaluate multiple systems using single LLM call for comparison
+        
+        Args:
+            systems_outputs: Dict mapping system names to their medical outputs
+        """
+        system_names = list(systems_outputs.keys())
+        total_queries = len(systems_outputs[system_names[0]])
+        
+        print(f"🧠 Multi-system comparison: {', '.join(system_names)}")
+        print(f"📊 Evaluating {total_queries} queries across {len(system_names)} systems...")
+        
+        try:
+            # Create comparison evaluation prompt
+            comparison_prompt = self.create_comparison_evaluation_prompt(systems_outputs)
+            
+            print(f"📝 Comparison prompt created ({len(comparison_prompt)} characters)")
+            print(f"🔄 Calling judge LLM for multi-system comparison...")
+            
+            # Single LLM call for all systems comparison
+            eval_start = time.time()
+            response = self.judge_llm.batch_evaluate(comparison_prompt)
+            eval_time = time.time() - eval_start
+            
+            # Extract response text
+            response_text = response.get('content', '') if isinstance(response, dict) else str(response)
+            
+            print(f"✅ Judge LLM completed comparison evaluation in {eval_time:.2f}s")
+            print(f"📄 Response length: {len(response_text)} characters")
+            
+            # Parse comparison response
+            results_by_system = self.parse_comparison_evaluation_response(response_text, systems_outputs)
+            
+            # Combine all results for storage
+            all_results = []
+            for system_name, system_results in results_by_system.items():
+                all_results.extend(system_results)
+                print(f"📊 {system_name.upper()}: {len(system_results)} evaluations parsed")
+            
+            self.evaluation_results.extend(all_results)
+            
+            return results_by_system
+            
+        except Exception as e:
+            print(f"❌ Multi-system evaluation failed: {e}")
+            
+            # Create error results for all systems
+            error_results = {}
+            for system_name, outputs in systems_outputs.items():
+                error_results[system_name] = []
+                for output in outputs:
+                    error_result = {
+                        "query": output.get('query', ''),
+                        "category": output.get('category', 'unknown'),
+                        "system_type": system_name,
+                        "actionability_score": 0.0,
+                        "evidence_score": 0.0,
+                        "evaluation_success": False,
+                        "error": str(e),
+                        "timestamp": datetime.now().isoformat()
+                    }
+                    error_results[system_name].append(error_result)
+                self.evaluation_results.extend(error_results[system_name])
+            
+            return error_results
+    
+    def calculate_judge_statistics(self) -> Dict[str, Any]:
+        """Calculate statistics for LLM judge evaluation"""
+        successful_results = [r for r in self.evaluation_results if r.get('evaluation_success')]
+        
+        if not successful_results:
+            return {
+                "category_results": {},
+                "overall_results": {
+                    "average_actionability": 0.0,
+                    "average_evidence": 0.0,
+                    "successful_evaluations": 0,
+                    "total_queries": len(self.evaluation_results)
+                },
+                "timestamp": datetime.now().isoformat()
+            }
+        
+        # Group by category
+        results_by_category = {"diagnosis": [], "treatment": [], "mixed": []}
+        
+        for result in successful_results:
+            category = result.get('category', 'unknown')
+            if category in results_by_category:
+                results_by_category[category].append(result)
+        
+        # Calculate category statistics
+        category_stats = {}
+        for category, results in results_by_category.items():
+            if results:
+                actionability_scores = [r['actionability_score'] for r in results]
+                evidence_scores = [r['evidence_score'] for r in results]
+                
+                category_stats[category] = {
+                    "average_actionability": sum(actionability_scores) / len(actionability_scores),
+                    "average_evidence": sum(evidence_scores) / len(evidence_scores),
+                    "query_count": len(results),
+                    "actionability_target_met": (sum(actionability_scores) / len(actionability_scores)) >= 0.7,
+                    "evidence_target_met": (sum(evidence_scores) / len(evidence_scores)) >= 0.75,
+                    "individual_actionability_scores": actionability_scores,
+                    "individual_evidence_scores": evidence_scores
+                }
+            else:
+                category_stats[category] = {
+                    "average_actionability": 0.0,
+                    "average_evidence": 0.0,
+                    "query_count": 0,
+                    "actionability_target_met": False,
+                    "evidence_target_met": False,
+                    "individual_actionability_scores": [],
+                    "individual_evidence_scores": []
+                }
+        
+        # Calculate overall statistics
+        all_actionability = [r['actionability_score'] for r in successful_results]
+        all_evidence = [r['evidence_score'] for r in successful_results]
+        
+        overall_stats = {
+            "average_actionability": sum(all_actionability) / len(all_actionability),
+            "average_evidence": sum(all_evidence) / len(all_evidence),
+            "successful_evaluations": len(successful_results),
+            "total_queries": len(self.evaluation_results),
+            "actionability_target_met": (sum(all_actionability) / len(all_actionability)) >= 0.7,
+            "evidence_target_met": (sum(all_evidence) / len(all_evidence)) >= 0.75
+        }
+        
+        return {
+            "category_results": category_stats,
+            "overall_results": overall_stats,
+            "timestamp": datetime.now().isoformat()
+        }
+    
+    def save_comparison_statistics(self, systems: List[str], filename: str = None) -> str:
+        """Save comparison evaluation statistics for multiple systems"""
+        stats = self.calculate_judge_statistics()
+        
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            systems_str = "_vs_".join(systems)
+            filename = f"judge_evaluation_comparison_{systems_str}_{timestamp}.json"
+        
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        filepath = results_dir / filename
+        
+        # Add comparison metadata
+        stats["comparison_metadata"] = {
+            "systems_compared": systems,
+            "comparison_type": "multi_system",
+            "timestamp": datetime.now().isoformat()
+        }
+        
+        # Add detailed system-specific results for chart generation
+        stats["detailed_system_results"] = {}
+        for system in systems:
+            system_results = [r for r in self.evaluation_results if r.get('system_type') == system and r.get('evaluation_success')]
+            stats["detailed_system_results"][system] = {
+                "results": system_results,
+                "query_count": len(system_results),
+                "avg_actionability": sum(r['actionability_score'] for r in system_results) / len(system_results) if system_results else 0.0,
+                "avg_evidence": sum(r['evidence_score'] for r in system_results) / len(system_results) if system_results else 0.0
+            }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(stats, f, indent=2, ensure_ascii=False)
+        
+        print(f"📊 Comparison evaluation statistics saved to: {filepath}")
+        return str(filepath)
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent LLM judge evaluation interface with multi-system support"""
+    
+    print("🧠 OnCall.ai LLM Judge Evaluator - Metrics 5-6 Multi-System Evaluation")
+    
+    # Print evaluation rubrics for reference
+    print_evaluation_rubrics()
+    
+    if len(sys.argv) < 2:
+        print("Usage: python metric5_6_llm_judge_evaluator.py [system1] or [system1,system2,...]")
+        print("  rag         - Evaluate RAG system medical outputs")
+        print("  direct      - Evaluate direct LLM medical outputs") 
+        print("  rag,direct  - Compare RAG vs Direct systems")
+        print("  system1,system2,system3  - Compare multiple systems")
+        sys.exit(1)
+    
+    # Parse systems from command line
+    systems_input = sys.argv[1]
+    systems = [s.strip() for s in systems_input.split(',')]
+    
+    # Initialize evaluator
+    evaluator = LLMJudgeEvaluator()
+    
+    try:
+        if len(systems) == 1:
+            # Single system evaluation (legacy mode)
+            system = systems[0]
+            print(f"\n🧪 Single System LLM Judge Evaluation: {system.upper()}")
+            
+            # Find and load medical outputs for single system
+            system_files = evaluator.find_medical_outputs_for_systems([system])
+            medical_outputs = evaluator.load_medical_outputs(system_files[system])
+            
+            if not medical_outputs:
+                print(f"❌ No medical outputs found for {system}")
+                sys.exit(1)
+            
+            print(f"📊 Evaluating {len(medical_outputs)} medical advice outputs")
+            print(f"🎯 Metrics: 5 (Actionability) + 6 (Evidence Quality)")
+            
+            # Convert to multi-system format for consistency
+            systems_outputs = {system: medical_outputs}
+            results_by_system = evaluator.evaluate_multiple_systems(systems_outputs)
+            
+            # Save results
+            stats_path = evaluator.save_comparison_statistics([system])
+            
+        else:
+            # Multi-system comparison evaluation  
+            print(f"\n🧪 Multi-System Comparison: {' vs '.join([s.upper() for s in systems])}")
+            
+            # Find and load medical outputs for all systems
+            system_files = evaluator.find_medical_outputs_for_systems(systems)
+            systems_outputs = {}
+            
+            for system in systems:
+                outputs = evaluator.load_medical_outputs(system_files[system])
+                if not outputs:
+                    print(f"❌ No medical outputs found for {system}")
+                    sys.exit(1)
+                systems_outputs[system] = outputs
+            
+            # Validate all systems have same number of queries
+            query_counts = [len(outputs) for outputs in systems_outputs.values()]
+            if len(set(query_counts)) > 1:
+                print(f"⚠️ Warning: Systems have different query counts: {dict(zip(systems, query_counts))}")
+            
+            # Validate systems processed same queries (for scientific comparison)
+            print(f"🔍 Validating query consistency across systems...")
+            if len(systems) > 1:
+                first_system_queries = [q['query'] for q in systems_outputs[systems[0]]]
+                for i, system in enumerate(systems[1:], 1):
+                    system_queries = [q['query'] for q in systems_outputs[system]]
+                    
+                    if first_system_queries != system_queries:
+                        print(f"⚠️ Warning: {systems[0]} and {system} processed different queries!")
+                        # Show first difference
+                        for j, (q1, q2) in enumerate(zip(first_system_queries, system_queries)):
+                            if q1 != q2:
+                                print(f"   Query {j+1} differs:")
+                                print(f"   {systems[0]}: {q1[:50]}...")
+                                print(f"   {system}: {q2[:50]}...")
+                                break
+                    else:
+                        print(f"✅ {systems[0]} and {system} processed identical queries")
+            
+            # Validate systems have different model types
+            model_types = set()
+            for system, outputs in systems_outputs.items():
+                if outputs:
+                    model_type = outputs[0].get('model_type', 'unknown')
+                    model_types.add(model_type)
+                    print(f"🏷️ {system.upper()} system model_type: {model_type}")
+            
+            if len(model_types) == 1:
+                print(f"⚠️ Warning: All systems have same model_type - this may not be a valid comparison!")
+            else:
+                print(f"✅ Systems have different model_types: {model_types}")
+            
+            print(f"📊 Comparing {len(systems)} systems with {min(query_counts)} queries each")
+            print(f"🎯 Metrics: 5 (Actionability) + 6 (Evidence Quality)")
+            print(f"⚡ Strategy: Single comparison call for maximum consistency")
+            
+            # Multi-system comparison evaluation
+            results_by_system = evaluator.evaluate_multiple_systems(systems_outputs)
+            
+            # Save comparison results
+            stats_path = evaluator.save_comparison_statistics(systems)
+        
+        # Print summary
+        print(f"\n📊 Generating evaluation analysis...")
+        stats = evaluator.calculate_judge_statistics()
+        overall_results = stats['overall_results']
+        
+        print(f"\n📊 === LLM JUDGE EVALUATION SUMMARY ===")
+        
+        if len(systems) == 1:
+            print(f"System: {systems[0].upper()}")
+        else:
+            print(f"Systems Compared: {' vs '.join([s.upper() for s in systems])}")
+        
+        print(f"Overall Performance:")
+        actionability_raw = overall_results['average_actionability'] * 10
+        evidence_raw = overall_results['average_evidence'] * 10
+        
+        print(f"   Average Actionability: {overall_results['average_actionability']:.3f} ({actionability_raw:.1f}/10)")
+        print(f"   • {get_rubric_description(int(actionability_raw), 'actionability')}")
+        print(f"   Average Evidence Quality: {overall_results['average_evidence']:.3f} ({evidence_raw:.1f}/10)")
+        print(f"   • {get_rubric_description(int(evidence_raw), 'evidence')}")
+        print(f"   Actionability Target (≥7.0): {'✅ Met' if overall_results['actionability_target_met'] else '❌ Not Met'}")
+        print(f"   Evidence Target (≥7.5): {'✅ Met' if overall_results['evidence_target_met'] else '❌ Not Met'}")
+        
+        # System-specific breakdown for multi-system comparison
+        if len(systems) > 1:
+            print(f"\nSystem Breakdown:")
+            for system in systems:
+                system_results = [r for r in evaluator.evaluation_results if r.get('system_type') == system and r.get('evaluation_success')]
+                if system_results:
+                    avg_action = sum(r['actionability_score'] for r in system_results) / len(system_results)
+                    avg_evidence = sum(r['evidence_score'] for r in system_results) / len(system_results)
+                    print(f"   {system.upper()}: Actionability={avg_action:.3f}, Evidence={avg_evidence:.3f} [{len(system_results)} queries]")
+        
+        print(f"\n✅ LLM judge evaluation complete!")
+        print(f"📊 Statistics: {stats_path}")
+        print(f"⚡ Efficiency: {overall_results['total_queries']} evaluations in 1 LLM call")
+        
+    except FileNotFoundError as e:
+        print(f"❌ {e}")
+        print(f"💡 Please run evaluators first:")
+        for system in systems:
+            if system == "rag":
+                print("   python latency_evaluator.py single_test_query.txt")
+            elif system == "direct":
+                print("   python direct_llm_evaluator.py single_test_query.txt")
+            else:
+                print(f"   python {system}_evaluator.py single_test_query.txt")
+    except Exception as e:
+        print(f"❌ Judge evaluation failed: {e}")

evaluation/metric7_8_precision_MRR.py

@@ -0,0 +1,402 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Precision & MRR Analyzer (Metrics 7-8)
+========================================================
+
+Specialized analyzer for calculating Precision@K and Mean Reciprocal Rank (MRR)
+using data collected from latency_evaluator.py comprehensive evaluation.
+
+IMPORTANT CHANGES - Angular Distance & Relevance Calculation:
+- DISTANCE METRIC: Uses Angular Distance from Annoy index (range: 0.0-1.0, smaller = more relevant)
+- RELEVANCE CONVERSION: relevance = 1.0 - (angular_distance²) / 2.0 (mathematical correct formula)
+- THRESHOLD ALIGNMENT: Aligned with Metric 3 relevance calculation standards
+- DISPLAY UPDATE: Changed from "Relevance: X" to "Angular Distance: X" for clarity
+
+METRICS CALCULATED:
+7. Precision@K (檢索精確率) - Proportion of relevant results in top-K retrieval
+8. Mean Reciprocal Rank (平均倒數排名) - Average reciprocal rank of first relevant result
+
+DESIGN PRINCIPLE:
+- Reuses comprehensive_details_*.json from latency_evaluator.py
+- Implements adaptive threshold based on query complexity
+- Query complexity determined by actual matched emergency keywords count
+- No additional LLM calls required
+
+Author: YanBo Chen  
+Date: 2025-08-04
+Updated: 2025-08-04 (Angular Distance alignment)
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any, Set
+from datetime import datetime
+from pathlib import Path
+import re
+import statistics
+
+# Relevance threshold constants for adaptive query complexity handling
+COMPLEX_QUERY_RELEVANCE_THRESHOLD = 0.65  # For queries with multiple emergency keywords
+SIMPLE_QUERY_RELEVANCE_THRESHOLD = 0.75   # For straightforward diagnostic queries
+
+class PrecisionMRRAnalyzer:
+    """Specialized analyzer for metrics 7-8 using existing comprehensive evaluation data"""
+    
+    def __init__(self):
+        """Initialize analyzer"""
+        print("🔧 Initializing Precision & MRR Analyzer...")
+        self.analysis_results = []
+        print("✅ Analyzer initialization complete")
+    
+    def load_comprehensive_data(self, filepath: str) -> List[Dict]:
+        """
+        Load comprehensive evaluation data from latency_evaluator.py output
+        
+        Args:
+            filepath: Path to comprehensive_details_*.json file
+            
+        Returns:
+            List of comprehensive evaluation results
+        """
+        try:
+            with open(filepath, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+            
+            comprehensive_results = data.get('comprehensive_results', [])
+            
+            print(f"📁 Loaded {len(comprehensive_results)} comprehensive evaluation results")
+            print(f"📊 Ready for precision/MRR analysis: {sum(1 for r in comprehensive_results if r.get('precision_mrr_ready'))}")
+            
+            return comprehensive_results
+            
+        except Exception as e:
+            print(f"❌ Failed to load comprehensive data: {e}")
+            return []
+    
+    def _is_complex_query(self, query: str, processed_results: List[Dict]) -> bool:
+        """
+        Determine query complexity based on actual matched emergency keywords
+        
+        Args:
+            query: Original query text
+            processed_results: Retrieval results with matched keywords
+            
+        Returns:
+            True if query is complex (should use lenient threshold)
+        """
+        # Collect unique emergency keywords actually found in retrieval results
+        unique_emergency_keywords = set()
+        
+        for result in processed_results:
+            if result.get('type') == 'emergency':
+                matched_keywords = result.get('matched', '')
+                if matched_keywords:
+                    keywords = [kw.strip() for kw in matched_keywords.split('|') if kw.strip()]
+                    unique_emergency_keywords.update(keywords)
+        
+        keyword_count = len(unique_emergency_keywords)
+        
+        # Business logic: 4+ different emergency keywords indicate complex case
+        is_complex = keyword_count >= 4
+        
+        print(f"   🧠 Query complexity: {'Complex' if is_complex else 'Simple'} ({keyword_count} emergency keywords)")
+        print(f"   🔑 Found keywords: {', '.join(list(unique_emergency_keywords)[:5])}")
+        
+        return is_complex
+    
+    def calculate_precision_mrr_single(self, query_data: Dict) -> Dict[str, Any]:
+        """
+        Calculate precision@K and MRR for single query
+        
+        Args:
+            query_data: Single query's comprehensive evaluation result
+            
+        Returns:
+            Precision and MRR metrics for this query
+        """
+        query = query_data['query']
+        category = query_data['category']
+        
+        # Extract processed results from pipeline data
+        pipeline_data = query_data.get('pipeline_data', {})
+        retrieval_results = pipeline_data.get('retrieval_results', {})
+        processed_results = retrieval_results.get('processed_results', [])
+        
+        print(f"🔍 Analyzing precision/MRR for: {query[:50]}...")
+        print(f"📋 Category: {category}, Results: {len(processed_results)}")
+        
+        if not processed_results:
+            return self._create_empty_precision_mrr_result(query, category)
+        
+        # Step 1: Determine query complexity
+        is_complex = self._is_complex_query(query, processed_results)
+        
+        # Step 2: Choose adaptive threshold (aligned with Metric 3 relevance standards)
+        threshold = COMPLEX_QUERY_RELEVANCE_THRESHOLD if is_complex else SIMPLE_QUERY_RELEVANCE_THRESHOLD  # Updated thresholds for complex/simple queries
+        
+        print(f"   🎯 Using relevance threshold: {threshold} ({'lenient' if is_complex else 'strict'})")
+        
+        # Step 3: Calculate relevance scores using correct angular distance formula
+        relevance_scores = []
+        for result in processed_results:
+            distance = result.get('distance', 1.0)
+            relevance = 1.0 - (distance**2) / 2.0  # Correct mathematical conversion
+            relevance_scores.append(relevance)
+        
+        # Step 4: Calculate Precision@K (aligned with Metric 3 thresholds)
+        relevant_count = sum(1 for score in relevance_scores if score >= threshold)
+        precision_at_k = relevant_count / len(processed_results)
+        
+        # Step 5: Calculate MRR
+        first_relevant_rank = None
+        for i, score in enumerate(relevance_scores, 1):
+            if score >= threshold:
+                first_relevant_rank = i
+                break
+        
+        mrr_score = (1.0 / first_relevant_rank) if first_relevant_rank else 0.0
+        
+        # Detailed analysis
+        result = {
+            "query": query,
+            "category": category,
+            "query_complexity": "complex" if is_complex else "simple",
+            "threshold_used": threshold,
+            
+            # Metric 7: Precision@K
+            "precision_at_k": precision_at_k,
+            "relevant_count": relevant_count,
+            "total_results": len(processed_results),
+            
+            # Metric 8: MRR
+            "mrr_score": mrr_score,
+            "first_relevant_rank": first_relevant_rank,
+            
+            # Supporting data
+            "relevance_scores": relevance_scores,
+            "avg_relevance": sum(relevance_scores) / len(relevance_scores),
+            "max_relevance": max(relevance_scores),
+            "min_relevance": min(relevance_scores),
+            
+            "timestamp": datetime.now().isoformat()
+        }
+        
+        print(f"   📊 Precision@{len(processed_results)}: {precision_at_k:.3f} ({relevant_count}/{len(processed_results)} relevant)")
+        print(f"   📊 MRR: {mrr_score:.3f} (first relevant at rank {first_relevant_rank})")
+        
+        return result
+    
+    def _create_empty_precision_mrr_result(self, query: str, category: str) -> Dict[str, Any]:
+        """Create empty result for failed queries"""
+        return {
+            "query": query,
+            "category": category,
+            "query_complexity": "unknown",
+            "threshold_used": 0.0,
+            "precision_at_k": 0.0,
+            "relevant_count": 0,
+            "total_results": 0,
+            "mrr_score": 0.0,
+            "first_relevant_rank": None,
+            "relevance_scores": [],
+            "timestamp": datetime.now().isoformat()
+        }
+    
+    def analyze_all_queries(self, comprehensive_results: List[Dict]) -> List[Dict]:
+        """
+        Analyze precision/MRR for all queries in comprehensive evaluation
+        
+        Args:
+            comprehensive_results: Results from latency_evaluator.py
+            
+        Returns:
+            List of precision/MRR analysis results
+        """
+        print(f"\n📊 Analyzing Precision@K and MRR for {len(comprehensive_results)} queries...")
+        
+        analysis_results = []
+        
+        for i, query_data in enumerate(comprehensive_results):
+            if not query_data.get('precision_mrr_ready'):
+                print(f"⏭️  Skipping query {i+1}: Not ready for precision/MRR analysis")
+                continue
+            
+            if not query_data.get('overall_success'):
+                print(f"⏭️  Skipping query {i+1}: Pipeline failed")
+                analysis_results.append(self._create_empty_precision_mrr_result(
+                    query_data['query'], 
+                    query_data['category']
+                ))
+                continue
+            
+            # Analyze this query
+            result = self.calculate_precision_mrr_single(query_data)
+            analysis_results.append(result)
+            
+            print("")  # Spacing between queries
+        
+        self.analysis_results = analysis_results
+        return analysis_results
+    
+    def calculate_statistics(self) -> Dict[str, Any]:
+        """Calculate comprehensive statistics for metrics 7-8"""
+        
+        if not self.analysis_results:
+            return {"error": "No analysis results available"}
+        
+        # Separate by complexity and category
+        stats = {
+            "overall_statistics": {},
+            "by_complexity": {"simple": {}, "complex": {}},
+            "by_category": {"diagnosis": {}, "treatment": {}, "mixed": {}},
+            "timestamp": datetime.now().isoformat()
+        }
+        
+        # Overall statistics
+        all_precision = [r['precision_at_k'] for r in self.analysis_results]
+        all_mrr = [r['mrr_score'] for r in self.analysis_results]
+        
+        stats["overall_statistics"] = {
+            "total_queries": len(self.analysis_results),
+            "avg_precision": statistics.mean(all_precision),
+            "avg_mrr": statistics.mean(all_mrr),
+            "precision_std": statistics.stdev(all_precision) if len(all_precision) > 1 else 0.0,
+            "mrr_std": statistics.stdev(all_mrr) if len(all_mrr) > 1 else 0.0
+        }
+        
+        # By complexity
+        for complexity in ["simple", "complex"]:
+            complexity_results = [r for r in self.analysis_results if r['query_complexity'] == complexity]
+            if complexity_results:
+                precision_scores = [r['precision_at_k'] for r in complexity_results]
+                mrr_scores = [r['mrr_score'] for r in complexity_results]
+                
+                stats["by_complexity"][complexity] = {
+                    "query_count": len(complexity_results),
+                    "avg_precision": statistics.mean(precision_scores),
+                    "avg_mrr": statistics.mean(mrr_scores),
+                    "avg_threshold": statistics.mean([r['threshold_used'] for r in complexity_results])
+                }
+        
+        # By category
+        for category in ["diagnosis", "treatment", "mixed"]:
+            category_results = [r for r in self.analysis_results if r['category'] == category]
+            if category_results:
+                precision_scores = [r['precision_at_k'] for r in category_results]
+                mrr_scores = [r['mrr_score'] for r in category_results]
+                
+                stats["by_category"][category] = {
+                    "query_count": len(category_results),
+                    "avg_precision": statistics.mean(precision_scores),
+                    "avg_mrr": statistics.mean(mrr_scores)
+                }
+        
+        return stats
+    
+    def save_results(self, filename: str = None) -> str:
+        """Save precision/MRR analysis results"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"precision_mrr_analysis_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        # Create output data
+        output_data = {
+            "analysis_metadata": {
+                "total_queries": len(self.analysis_results),
+                "analysis_type": "precision_mrr_metrics_7_8",
+                "timestamp": datetime.now().isoformat(),
+                "adaptive_threshold": True
+            },
+            "detailed_results": self.analysis_results,
+            "statistics": self.calculate_statistics()
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(output_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"📊 Precision/MRR analysis saved to: {filepath}")
+        return str(filepath)
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent precision/MRR analysis interface"""
+    
+    print("📊 OnCall.ai Precision & MRR Analyzer - Metrics 7-8")
+    
+    if len(sys.argv) > 1:
+        comprehensive_file = sys.argv[1]
+    else:
+        # Look for latest comprehensive_details file
+        results_dir = Path(__file__).parent / "results"
+        if results_dir.exists():
+            comprehensive_files = list(results_dir.glob("comprehensive_details_*.json"))
+            if comprehensive_files:
+                comprehensive_file = str(sorted(comprehensive_files)[-1])  # Latest file
+                print(f"📁 Using latest comprehensive file: {comprehensive_file}")
+            else:
+                print("❌ No comprehensive_details_*.json files found")
+                print("Please run latency_evaluator.py first to generate comprehensive data")
+                sys.exit(1)
+        else:
+            print("❌ Results directory not found")
+            sys.exit(1)
+    
+    if not os.path.exists(comprehensive_file):
+        print(f"❌ Comprehensive file not found: {comprehensive_file}")
+        print("Usage: python precision_MRR.py [comprehensive_details_file.json]")
+        sys.exit(1)
+    
+    # Initialize analyzer
+    analyzer = PrecisionMRRAnalyzer()
+    
+    # Load comprehensive data from latency_evaluator.py
+    comprehensive_results = analyzer.load_comprehensive_data(comprehensive_file)
+    
+    if not comprehensive_results:
+        print("❌ No comprehensive data loaded")
+        sys.exit(1)
+    
+    # Analyze precision/MRR for all queries
+    analysis_results = analyzer.analyze_all_queries(comprehensive_results)
+    
+    # Calculate and display statistics
+    statistics_result = analyzer.calculate_statistics()
+    
+    print(f"\n📊 === PRECISION & MRR ANALYSIS SUMMARY ===")
+    
+    overall_stats = statistics_result['overall_statistics']
+    print(f"\nOVERALL METRICS:")
+    print(f"   Precision@K: {overall_stats['avg_precision']:.3f} (±{overall_stats['precision_std']:.3f})")
+    print(f"   MRR: {overall_stats['avg_mrr']:.3f} (±{overall_stats['mrr_std']:.3f})")
+    print(f"   Total Queries: {overall_stats['total_queries']}")
+    
+    # Complexity-based statistics
+    complexity_stats = statistics_result['by_complexity']
+    print(f"\nBY COMPLEXITY:")
+    for complexity, stats in complexity_stats.items():
+        if stats:
+            print(f"   {complexity.title()}: Precision={stats['avg_precision']:.3f}, MRR={stats['avg_mrr']:.3f} "
+                  f"(threshold={stats['avg_threshold']:.2f}, n={stats['query_count']})")
+    
+    # Category-based statistics  
+    category_stats = statistics_result['by_category']
+    print(f"\nBY CATEGORY:")
+    for category, stats in category_stats.items():
+        if stats:
+            print(f"   {category.title()}: Precision={stats['avg_precision']:.3f}, MRR={stats['avg_mrr']:.3f} "
+                  f"(n={stats['query_count']})")
+    
+    # Save results
+    saved_path = analyzer.save_results()
+    
+    print(f"\n✅ Precision & MRR analysis complete!")
+    print(f"📁 Results saved to: {saved_path}")
+    print(f"\n💡 Next step: Create precision_mrr_chart_generator.py for visualization")

evaluation/metric7_8_precision_mrr_chart_generator.py

@@ -0,0 +1,586 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Precision & MRR Chart Generator (Metrics 7-8)
+===============================================================
+
+Generates comprehensive Precision@K and MRR analysis charts from saved analysis results.
+Reads JSON files produced by metric7_8_precision_MRR.py and creates visualizations.
+
+Charts generated:
+1. Precision@K comparison by category and complexity
+2. MRR comparison by category and complexity  
+3. Combined metrics heatmap
+4. Threshold impact analysis
+5. Detailed statistics tables
+
+No LLM calls - pure data visualization.
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any
+from datetime import datetime
+from pathlib import Path
+import glob
+
+# Visualization imports
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+import numpy as np
+
+
+class PrecisionMRRChartGenerator:
+    """Generate charts from precision/MRR analysis results - no LLM dependency"""
+    
+    def __init__(self):
+        """Initialize chart generator"""
+        print("📈 Initializing Precision & MRR Chart Generator...")
+        
+        # Set up professional chart style
+        plt.style.use('default')
+        sns.set_palette("husl")
+        
+        print("✅ Chart Generator ready")
+    
+    def load_latest_analysis(self, results_dir: str = None) -> Dict[str, Any]:
+        """
+        Load the most recent precision/MRR analysis file
+        
+        Args:
+            results_dir: Directory containing analysis files
+        """
+        if results_dir is None:
+            results_dir = Path(__file__).parent / "results"
+        
+        analysis_files = glob.glob(str(results_dir / "precision_mrr_analysis_*.json"))
+        
+        if not analysis_files:
+            raise FileNotFoundError("No precision_mrr_analysis_*.json files found. Run metric7_8_precision_MRR.py first.")
+        
+        latest_file = max(analysis_files, key=os.path.getctime)
+        print(f"📁 Loading latest analysis: {latest_file}")
+        
+        with open(latest_file, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    
+    def create_precision_comparison_chart(self, analysis_data: Dict, save_path: str = None) -> str:
+        """Create Precision@K comparison chart"""
+        
+        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))
+        
+        # Chart 1: Precision by Category
+        category_stats = analysis_data['statistics']['by_category']
+        categories = []
+        precisions = []
+        
+        for category, stats in category_stats.items():
+            if stats:
+                categories.append(category.title())
+                precisions.append(stats['avg_precision'])
+        
+        if categories:
+            bars1 = ax1.bar(categories, precisions, alpha=0.8, color=['#1f77b4', '#ff7f0e', '#d62728'])
+            ax1.set_title('Precision@K by Query Category', fontweight='bold')
+            ax1.set_ylabel('Precision@K')
+            ax1.set_xlabel('Query Category')
+            ax1.set_ylim(0, 1.0)
+            ax1.grid(True, alpha=0.3)
+            
+            # Add value labels
+            for bar, precision in zip(bars1, precisions):
+                height = bar.get_height()
+                ax1.text(bar.get_x() + bar.get_width()/2., height + 0.01,
+                        f'{precision:.3f}', ha='center', va='bottom', fontweight='bold')
+        
+        # Chart 2: Precision by Complexity
+        complexity_stats = analysis_data['statistics']['by_complexity']
+        complexities = []
+        comp_precisions = []
+        
+        for complexity, stats in complexity_stats.items():
+            if stats:
+                complexities.append(complexity.title())
+                comp_precisions.append(stats['avg_precision'])
+        
+        if complexities:
+            bars2 = ax2.bar(complexities, comp_precisions, alpha=0.8, color=['#2ca02c', '#d62728'])
+            ax2.set_title('Precision@K by Query Complexity', fontweight='bold')
+            ax2.set_ylabel('Precision@K')
+            ax2.set_xlabel('Query Complexity')
+            ax2.set_ylim(0, 1.0)
+            ax2.grid(True, alpha=0.3)
+            
+            # Add value labels and threshold info
+            for bar, precision, complexity in zip(bars2, comp_precisions, complexities):
+                height = bar.get_height()
+                threshold = 0.15 if complexity.lower() == 'complex' else 0.25
+                ax2.text(bar.get_x() + bar.get_width()/2., height + 0.01,
+                        f'{precision:.3f}\n(T={threshold})', ha='center', va='bottom', 
+                        fontweight='bold', fontsize=9)
+        
+        plt.tight_layout()
+        
+        # Save chart
+        if save_path is None:
+            save_path = Path(__file__).parent / "charts" / f"precision_comparison_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
+        
+        save_path = Path(save_path)
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+        plt.savefig(save_path, dpi=300, bbox_inches='tight')
+        plt.close()
+        
+        print(f"📊 Precision comparison chart saved: {save_path}")
+        return str(save_path)
+    
+    def create_mrr_comparison_chart(self, analysis_data: Dict, save_path: str = None) -> str:
+        """Create MRR comparison chart"""
+        
+        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))
+        
+        # Chart 1: MRR by Category
+        category_stats = analysis_data['statistics']['by_category']
+        categories = []
+        mrr_scores = []
+        
+        for category, stats in category_stats.items():
+            if stats:
+                categories.append(category.title())
+                mrr_scores.append(stats['avg_mrr'])
+        
+        if categories:
+            bars1 = ax1.bar(categories, mrr_scores, alpha=0.8, color=['#9467bd', '#8c564b', '#e377c2'])
+            ax1.set_title('Mean Reciprocal Rank by Query Category', fontweight='bold')
+            ax1.set_ylabel('MRR Score')
+            ax1.set_xlabel('Query Category')
+            ax1.set_ylim(0, 1.0)
+            ax1.grid(True, alpha=0.3)
+            
+            # Add value labels
+            for bar, mrr in zip(bars1, mrr_scores):
+                height = bar.get_height()
+                ax1.text(bar.get_x() + bar.get_width()/2., height + 0.01,
+                        f'{mrr:.3f}', ha='center', va='bottom', fontweight='bold')
+        
+        # Chart 2: MRR by Complexity
+        complexity_stats = analysis_data['statistics']['by_complexity']
+        complexities = []
+        comp_mrr = []
+        
+        for complexity, stats in complexity_stats.items():
+            if stats:
+                complexities.append(complexity.title())
+                comp_mrr.append(stats['avg_mrr'])
+        
+        if complexities:
+            bars2 = ax2.bar(complexities, comp_mrr, alpha=0.8, color=['#17becf', '#bcbd22'])
+            ax2.set_title('MRR by Query Complexity', fontweight='bold')
+            ax2.set_ylabel('MRR Score')
+            ax2.set_xlabel('Query Complexity')
+            ax2.set_ylim(0, 1.0)
+            ax2.grid(True, alpha=0.3)
+            
+            # Add value labels
+            for bar, mrr in zip(bars2, comp_mrr):
+                height = bar.get_height()
+                ax2.text(bar.get_x() + bar.get_width()/2., height + 0.01,
+                        f'{mrr:.3f}', ha='center', va='bottom', fontweight='bold')
+        
+        plt.tight_layout()
+        
+        # Save chart
+        if save_path is None:
+            save_path = Path(__file__).parent / "charts" / f"mrr_comparison_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
+        
+        save_path = Path(save_path)
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+        plt.savefig(save_path, dpi=300, bbox_inches='tight')
+        plt.close()
+        
+        print(f"📊 MRR comparison chart saved: {save_path}")
+        return str(save_path)
+    
+    def create_combined_metrics_heatmap(self, analysis_data: Dict, save_path: str = None) -> str:
+        """Create combined precision/MRR heatmap"""
+        
+        # Prepare data for heatmap
+        detailed_results = analysis_data.get('detailed_results', [])
+        
+        if not detailed_results:
+            print("⚠️ No detailed results for heatmap")
+            return ""
+        
+        # Create DataFrame for heatmap
+        heatmap_data = []
+        for result in detailed_results:
+            heatmap_data.append({
+                'Category': result['category'].title(),
+                'Complexity': result['query_complexity'].title(),
+                'Precision@K': result['precision_at_k'],
+                'MRR': result['mrr_score'],
+                'Threshold': result['threshold_used']
+            })
+        
+        df = pd.DataFrame(heatmap_data)
+        
+        # Create pivot table for heatmap
+        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))
+        
+        # Precision heatmap
+        precision_pivot = df.pivot_table(values='Precision@K', index='Category', columns='Complexity', aggfunc='mean')
+        sns.heatmap(precision_pivot, annot=True, fmt='.3f', cmap='YlOrRd', ax=ax1, 
+                   cbar_kws={'label': 'Precision@K'}, vmin=0, vmax=1)
+        ax1.set_title('Precision@K Heatmap\n(Category vs Complexity)', fontweight='bold')
+        
+        # MRR heatmap  
+        mrr_pivot = df.pivot_table(values='MRR', index='Category', columns='Complexity', aggfunc='mean')
+        sns.heatmap(mrr_pivot, annot=True, fmt='.3f', cmap='YlGnBu', ax=ax2,
+                   cbar_kws={'label': 'MRR Score'}, vmin=0, vmax=1)
+        ax2.set_title('MRR Heatmap\n(Category vs Complexity)', fontweight='bold')
+        
+        plt.tight_layout()
+        
+        # Save chart
+        if save_path is None:
+            save_path = Path(__file__).parent / "charts" / f"precision_mrr_heatmap_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
+        
+        save_path = Path(save_path)
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+        plt.savefig(save_path, dpi=300, bbox_inches='tight')
+        plt.close()
+        
+        print(f"📊 Combined metrics heatmap saved: {save_path}")
+        return str(save_path)
+    
+    def create_threshold_impact_chart(self, analysis_data: Dict, save_path: str = None) -> str:
+        """Create threshold impact analysis chart"""
+        
+        detailed_results = analysis_data.get('detailed_results', [])
+        
+        if not detailed_results:
+            print("⚠️ No detailed results for threshold analysis")
+            return ""
+        
+        # Group by complexity and calculate average relevance
+        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))
+        
+        # Prepare data
+        simple_queries = [r for r in detailed_results if r['query_complexity'] == 'simple']
+        complex_queries = [r for r in detailed_results if r['query_complexity'] == 'complex']
+        
+        # Chart 1: Relevance distribution for different complexities
+        if simple_queries:
+            simple_relevances = []
+            for query in simple_queries:
+                simple_relevances.extend(query.get('relevance_scores', []))
+            
+            ax1.hist(simple_relevances, bins=10, alpha=0.7, label=f'Simple (T=0.25)', color='#2ca02c', density=True)
+            ax1.axvline(x=0.25, color='#2ca02c', linestyle='--', linewidth=2, label='Simple Threshold')
+        
+        if complex_queries:
+            complex_relevances = []
+            for query in complex_queries:
+                complex_relevances.extend(query.get('relevance_scores', []))
+            
+            ax1.hist(complex_relevances, bins=10, alpha=0.7, label=f'Complex (T=0.15)', color='#d62728', density=True)
+            ax1.axvline(x=0.15, color='#d62728', linestyle='--', linewidth=2, label='Complex Threshold')
+        
+        ax1.set_title('Relevance Score Distribution\nby Query Complexity', fontweight='bold')
+        ax1.set_xlabel('Relevance Score')
+        ax1.set_ylabel('Density')
+        ax1.legend()
+        ax1.grid(True, alpha=0.3)
+        
+        # Chart 2: Metrics comparison
+        complexity_stats = analysis_data['statistics']['by_complexity']
+        
+        complexities = []
+        precisions = []
+        mrrs = []
+        thresholds = []
+        
+        for complexity, stats in complexity_stats.items():
+            if stats:
+                complexities.append(complexity.title())
+                precisions.append(stats['avg_precision'])
+                mrrs.append(stats['avg_mrr'])
+                thresholds.append(stats['avg_threshold'])
+        
+        x = np.arange(len(complexities))
+        width = 0.35
+        
+        bars1 = ax2.bar(x - width/2, precisions, width, label='Precision@K', alpha=0.8, color='#ff7f0e')
+        bars2 = ax2.bar(x + width/2, mrrs, width, label='MRR', alpha=0.8, color='#1f77b4')
+        
+        ax2.set_title('Metrics Comparison by Complexity\n(with Adaptive Thresholds)', fontweight='bold')
+        ax2.set_ylabel('Score')
+        ax2.set_xlabel('Query Complexity')
+        ax2.set_xticks(x)
+        ax2.set_xticklabels(complexities)
+        ax2.legend()
+        ax2.grid(True, alpha=0.3)
+        ax2.set_ylim(0, 1.0)
+        
+        # Add value labels
+        for bars, values, thresholds_vals in [(bars1, precisions, thresholds), (bars2, mrrs, thresholds)]:
+            for bar, value, threshold in zip(bars, values, thresholds_vals):
+                height = bar.get_height()
+                ax2.text(bar.get_x() + bar.get_width()/2., height + 0.01,
+                        f'{value:.3f}', ha='center', va='bottom', fontweight='bold', fontsize=9)
+        
+        plt.tight_layout()
+        
+        # Save chart
+        if save_path is None:
+            save_path = Path(__file__).parent / "charts" / f"threshold_impact_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
+        
+        save_path = Path(save_path)
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+        plt.savefig(save_path, dpi=300, bbox_inches='tight')
+        plt.close()
+        
+        print(f"📊 Threshold impact chart saved: {save_path}")
+        return str(save_path)
+    
+    def create_detailed_analysis_table(self, analysis_data: Dict, save_path: str = None) -> str:
+        """Create detailed statistics table"""
+        
+        fig, ax = plt.subplots(figsize=(12, 8))
+        ax.axis('tight')
+        ax.axis('off')
+        
+        # Prepare table data
+        table_data = []
+        
+        # Overall statistics
+        overall_stats = analysis_data['statistics']['overall_statistics']
+        table_data.append(['OVERALL METRICS', '', '', '', ''])
+        table_data.append(['Total Queries', str(overall_stats['total_queries']), '', '', ''])
+        table_data.append(['Avg Precision@K', f"{overall_stats['avg_precision']:.3f}", 
+                          f"±{overall_stats['precision_std']:.3f}", '', ''])
+        table_data.append(['Avg MRR', f"{overall_stats['avg_mrr']:.3f}", 
+                          f"±{overall_stats['mrr_std']:.3f}", '', ''])
+        table_data.append(['', '', '', '', ''])
+        
+        # By category
+        table_data.append(['BY CATEGORY', 'Queries', 'Precision@K', 'MRR', 'Notes'])
+        category_stats = analysis_data['statistics']['by_category']
+        for category, stats in category_stats.items():
+            if stats:
+                table_data.append([
+                    category.title(),
+                    str(stats['query_count']),
+                    f"{stats['avg_precision']:.3f}",
+                    f"{stats['avg_mrr']:.3f}",
+                    ''
+                ])
+        
+        table_data.append(['', '', '', '', ''])
+        
+        # By complexity
+        table_data.append(['BY COMPLEXITY', 'Queries', 'Precision@K', 'MRR', 'Threshold'])
+        complexity_stats = analysis_data['statistics']['by_complexity']
+        for complexity, stats in complexity_stats.items():
+            if stats:
+                table_data.append([
+                    complexity.title(),
+                    str(stats['query_count']),
+                    f"{stats['avg_precision']:.3f}",
+                    f"{stats['avg_mrr']:.3f}",
+                    f"{stats['avg_threshold']:.2f}"
+                ])
+        
+        # Create table
+        table = ax.table(cellText=table_data,
+                        colLabels=['Metric', 'Value 1', 'Value 2', 'Value 3', 'Value 4'],
+                        cellLoc='center',
+                        loc='center',
+                        bbox=[0, 0, 1, 1])
+        
+        # Style the table
+        table.auto_set_font_size(False)
+        table.set_fontsize(10)
+        table.scale(1, 2)
+        
+        # Header styling
+        for i in range(5):
+            table[(0, i)].set_facecolor('#40466e')
+            table[(0, i)].set_text_props(weight='bold', color='white')
+        
+        # Section headers styling
+        for i, row in enumerate(table_data):
+            if row[0] in ['OVERALL METRICS', 'BY CATEGORY', 'BY COMPLEXITY']:
+                table[(i+1, 0)].set_facecolor('#1f77b4')
+                table[(i+1, 0)].set_text_props(weight='bold', color='white')
+        
+        plt.title('Precision@K & MRR Detailed Analysis\nMetrics 7-8 Statistics', 
+                 fontweight='bold', fontsize=14, pad=20)
+        
+        # Save chart
+        if save_path is None:
+            save_path = Path(__file__).parent / "charts" / f"precision_mrr_table_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
+        
+        save_path = Path(save_path)
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+        plt.savefig(save_path, dpi=300, bbox_inches='tight')
+        plt.close()
+        
+        print(f"📊 Detailed analysis table saved: {save_path}")
+        return str(save_path)
+    
+    def create_individual_query_analysis(self, analysis_data: Dict, save_path: str = None) -> str:
+        """Create individual query analysis chart"""
+        
+        detailed_results = analysis_data.get('detailed_results', [])
+        
+        if not detailed_results:
+            print("⚠️ No detailed results for individual analysis")
+            return ""
+        
+        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(14, 10))
+        
+        # Prepare data
+        query_indices = []
+        precisions = []
+        mrrs = []
+        colors = []
+        labels = []
+        
+        for i, result in enumerate(detailed_results):
+            query_indices.append(i + 1)
+            precisions.append(result['precision_at_k'])
+            mrrs.append(result['mrr_score'])
+            
+            # Color by complexity
+            if result['query_complexity'] == 'complex':
+                colors.append('#d62728')  # Red for complex
+            else:
+                colors.append('#2ca02c')  # Green for simple
+            
+            # Create short label
+            query_short = result['query'][:30] + "..." if len(result['query']) > 30 else result['query']
+            category = result['category'][:4].upper()
+            labels.append(f"{category}\n{query_short}")
+        
+        # Chart 1: Precision@K for each query
+        bars1 = ax1.bar(query_indices, precisions, color=colors, alpha=0.8)
+        ax1.set_title('Precision@K by Individual Query', fontweight='bold')
+        ax1.set_ylabel('Precision@K')
+        ax1.set_xlabel('Query Index')
+        ax1.set_ylim(0, 1.0)
+        ax1.grid(True, alpha=0.3)
+        
+        # Add value labels
+        for bar, precision in zip(bars1, precisions):
+            height = bar.get_height()
+            ax1.text(bar.get_x() + bar.get_width()/2., height + 0.01,
+                    f'{precision:.2f}', ha='center', va='bottom', fontsize=8)
+        
+        # Chart 2: MRR for each query
+        bars2 = ax2.bar(query_indices, mrrs, color=colors, alpha=0.8)
+        ax2.set_title('MRR by Individual Query', fontweight='bold')
+        ax2.set_ylabel('MRR Score')
+        ax2.set_xlabel('Query Index')
+        ax2.set_ylim(0, 1.0)
+        ax2.grid(True, alpha=0.3)
+        
+        # Add value labels
+        for bar, mrr in zip(bars2, mrrs):
+            height = bar.get_height()
+            ax2.text(bar.get_x() + bar.get_width()/2., height + 0.01,
+                    f'{mrr:.2f}', ha='center', va='bottom', fontsize=8)
+        
+        # Add legend
+        from matplotlib.patches import Patch
+        legend_elements = [
+            Patch(facecolor='#2ca02c', alpha=0.8, label='Simple Query (T=0.25)'),
+            Patch(facecolor='#d62728', alpha=0.8, label='Complex Query (T=0.15)')
+        ]
+        ax1.legend(handles=legend_elements, loc='upper right')
+        
+        plt.tight_layout()
+        
+        # Save chart
+        if save_path is None:
+            save_path = Path(__file__).parent / "charts" / f"individual_query_analysis_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
+        
+        save_path = Path(save_path)
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+        plt.savefig(save_path, dpi=300, bbox_inches='tight')
+        plt.close()
+        
+        print(f"📊 Individual query analysis saved: {save_path}")
+        return str(save_path)
+    
+    def generate_all_charts(self, analysis_data: Dict = None) -> Dict[str, str]:
+        """Generate all precision/MRR charts"""
+        
+        if analysis_data is None:
+            analysis_data = self.load_latest_analysis()
+        
+        print(f"\n📈 Generating all Precision & MRR charts...")
+        
+        saved_charts = {}
+        
+        # Generate all chart types
+        try:
+            saved_charts['precision_comparison'] = self.create_precision_comparison_chart(analysis_data)
+            saved_charts['mrr_comparison'] = self.create_mrr_comparison_chart(analysis_data)
+            saved_charts['combined_heatmap'] = self.create_combined_metrics_heatmap(analysis_data)
+            saved_charts['threshold_impact'] = self.create_threshold_impact_chart(analysis_data)
+            saved_charts['individual_analysis'] = self.create_individual_query_analysis(analysis_data)
+            
+        except Exception as e:
+            print(f"❌ Error generating charts: {e}")
+            return {"error": str(e)}
+        
+        print(f"\n✅ All precision/MRR charts generated successfully!")
+        print(f"📁 Charts saved to: evaluation/charts/")
+        
+        return saved_charts
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Generate precision/MRR charts from analysis results"""
+    
+    print("📈 OnCall.ai Precision & MRR Chart Generator - Metrics 7-8")
+    
+    if len(sys.argv) > 1:
+        analysis_file = sys.argv[1]
+        
+        if not os.path.exists(analysis_file):
+            print(f"❌ Analysis file not found: {analysis_file}")
+            sys.exit(1)
+    else:
+        analysis_file = None  # Will use latest file
+    
+    # Initialize generator
+    generator = PrecisionMRRChartGenerator()
+    
+    try:
+        # Load analysis data
+        if analysis_file:
+            with open(analysis_file, 'r', encoding='utf-8') as f:
+                analysis_data = json.load(f)
+            print(f"📁 Using specified analysis file: {analysis_file}")
+        else:
+            analysis_data = generator.load_latest_analysis()
+        
+        # Generate all charts
+        saved_charts = generator.generate_all_charts(analysis_data)
+        
+        if 'error' not in saved_charts:
+            print(f"\n📊 === PRECISION & MRR CHART GENERATION SUMMARY ===")
+            for chart_type, filepath in saved_charts.items():
+                print(f"   📈 {chart_type.replace('_', ' ').title()}: {filepath}")
+            
+            print(f"\n💡 Charts ready for analysis and presentation!")
+        
+    except Exception as e:
+        print(f"❌ Chart generation failed: {e}")
+        sys.exit(1)

evaluation/old/coverage_evaluator.py

@@ -0,0 +1,560 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Retrieval Coverage Evaluator (Metric 4)
+==========================================================
+
+Evaluates how well generated medical advice utilizes retrieved content
+Automatic evaluation using keyword overlap analysis with optional LLM sampling
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any, Set
+from datetime import datetime
+from pathlib import Path
+import re
+
+# Add project path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import existing system components
+try:
+    from user_prompt import UserPromptProcessor
+    from retrieval import BasicRetrievalSystem
+    from llm_clients import llm_Med42_70BClient
+    from generation import MedicalAdviceGenerator
+except ImportError as e:
+    print(f"❌ Import failed: {e}")
+    print("Please ensure running from project root directory")
+    sys.exit(1)
+
+
+class CoverageEvaluator:
+    """Retrieval coverage evaluator using keyword overlap analysis"""
+    
+    def __init__(self):
+        """Initialize system components for coverage testing"""
+        print("🔧 Initializing Coverage Evaluator...")
+        
+        # Initialize full pipeline components (needed for advice generation)
+        self.llm_client = llm_Med42_70BClient()
+        self.retrieval_system = BasicRetrievalSystem()
+        self.user_prompt_processor = UserPromptProcessor(
+            llm_client=self.llm_client,
+            retrieval_system=self.retrieval_system
+        )
+        self.medical_generator = MedicalAdviceGenerator(llm_client=self.llm_client)
+        
+        # Results accumulation
+        self.coverage_results = []
+        
+        print("✅ Coverage Evaluator initialization complete")
+    
+    def extract_medical_keywords(self, text: str) -> Set[str]:
+        """
+        Extract medical keywords from text for coverage analysis
+        
+        Uses medical terminology patterns and common medical terms
+        """
+        if not text:
+            return set()
+        
+        medical_keywords = set()
+        text_lower = text.lower()
+        
+        # Medical terminology patterns
+        patterns = [
+            r'\b[a-z]+(?:osis|itis|pathy|emia|uria|gram|scopy)\b',  # Medical suffixes
+            r'\b(?:cardio|neuro|pulmo|gastro|hepato|nephro)[a-z]+\b',  # Medical prefixes
+            r'\b(?:diagnosis|treatment|therapy|intervention|management)\b',  # Medical actions
+            r'\b(?:patient|symptom|condition|disease|disorder|syndrome)\b',  # Medical entities
+            r'\b(?:acute|chronic|severe|mild|moderate|emergency)\b',  # Medical descriptors
+            r'\b[a-z]+(?:al|ic|ous|ive)\s+(?:pain|failure|infection|injury)\b',  # Compound terms
+            r'\b(?:ecg|ekg|ct|mri|x-ray|ultrasound|biopsy)\b',  # Medical procedures
+            r'\b\d+\s*(?:mg|ml|units|hours|days|minutes)\b',  # Dosages and timeframes
+        ]
+        
+        for pattern in patterns:
+            matches = re.findall(pattern, text_lower)
+            medical_keywords.update(match.strip() for match in matches)
+        
+        # Additional common medical terms
+        common_medical_terms = [
+            'blood', 'pressure', 'heart', 'chest', 'pain', 'stroke', 'seizure',
+            'emergency', 'hospital', 'monitor', 'assess', 'evaluate', 'immediate',
+            'protocol', 'guideline', 'recommendation', 'risk', 'factor'
+        ]
+        
+        for term in common_medical_terms:
+            if term in text_lower:
+                medical_keywords.add(term)
+        
+        # Filter out very short terms and common words
+        filtered_keywords = {
+            kw for kw in medical_keywords 
+            if len(kw) > 2 and kw not in ['the', 'and', 'for', 'with', 'are', 'can', 'may']
+        }
+        
+        return filtered_keywords
+    
+    def calculate_coverage_score(self, generated_advice: str, retrieval_results: List[Dict]) -> Dict[str, Any]:
+        """
+        Calculate coverage score based on keyword overlap between advice and retrieved docs
+        
+        Args:
+            generated_advice: Generated medical advice text
+            retrieval_results: List of retrieved documents
+        """
+        if not generated_advice or not retrieval_results:
+            return {
+                "coverage_score": 0.0,
+                "matched_keywords": [],
+                "advice_keywords": [],
+                "source_keywords": [],
+                "coverage_details": []
+            }
+        
+        # Extract keywords from generated advice
+        advice_keywords = self.extract_medical_keywords(generated_advice)
+        
+        # Extract keywords from all retrieved documents
+        all_source_keywords = set()
+        coverage_details = []
+        
+        for i, doc in enumerate(retrieval_results):
+            doc_content = doc.get('content', '') or doc.get('text', '')
+            doc_keywords = self.extract_medical_keywords(doc_content)
+            all_source_keywords.update(doc_keywords)
+            
+            # Calculate overlap for this specific document
+            doc_overlap = advice_keywords.intersection(doc_keywords)
+            doc_coverage = len(doc_overlap) / len(doc_keywords) if doc_keywords else 0.0
+            
+            coverage_details.append({
+                "doc_index": i,
+                "doc_snippet": doc_content[:100] + "...",
+                "doc_keywords_count": len(doc_keywords),
+                "matched_keywords_count": len(doc_overlap),
+                "doc_coverage_ratio": doc_coverage,
+                "matched_keywords": list(doc_overlap)[:10]  # Limit for readability
+            })
+        
+        # Calculate overall coverage
+        matched_keywords = advice_keywords.intersection(all_source_keywords)
+        coverage_score = len(matched_keywords) / len(all_source_keywords) if all_source_keywords else 0.0
+        
+        return {
+            "coverage_score": coverage_score,
+            "matched_keywords": list(matched_keywords),
+            "advice_keywords": list(advice_keywords),
+            "source_keywords": list(all_source_keywords),
+            "advice_keywords_count": len(advice_keywords),
+            "source_keywords_count": len(all_source_keywords),
+            "matched_keywords_count": len(matched_keywords),
+            "coverage_percentage": coverage_score * 100,
+            "meets_threshold": coverage_score >= 0.6,
+            "coverage_details": coverage_details
+        }
+    
+    def evaluate_single_coverage(self, query: str, category: str = "unknown") -> Dict[str, Any]:
+        """
+        Evaluate retrieval coverage for a single query
+        
+        Requires full pipeline: extraction → retrieval → generation → coverage analysis
+        
+        Args:
+            query: Medical query to test
+            category: Query category (diagnosis/treatment/mixed)
+        """
+        print(f"🔍 Testing coverage for: {query[:50]}...")
+        print(f"📋 Category: {category}")
+        
+        try:
+            # Step 1: Extract condition
+            condition_result = self.user_prompt_processor.extract_condition_keywords(query)
+            
+            # Step 2: Perform retrieval
+            search_query = f"{condition_result.get('emergency_keywords', '')} {condition_result.get('treatment_keywords', '')}".strip()
+            if not search_query:
+                search_query = condition_result.get('condition', query)
+            
+            retrieval_start = datetime.now()
+            retrieval_results = self.retrieval_system.search(search_query, top_k=5)
+            retrieval_time = (datetime.now() - retrieval_start).total_seconds()
+            
+            processed_results = retrieval_results.get('processed_results', [])
+            
+            if not processed_results:
+                result = {
+                    "query": query,
+                    "category": category,
+                    "search_query": search_query,
+                    "pipeline_success": False,
+                    "coverage_score": 0.0,
+                    "error": "No retrieval results",
+                    "timestamp": datetime.now().isoformat()
+                }
+                
+                self.coverage_results.append(result)
+                print(f"   ❌ No retrieval results for coverage analysis")
+                return result
+            
+            # Step 3: Generate medical advice
+            generation_start = datetime.now()
+            intention = self._detect_query_intention(query)
+            medical_advice_result = self.medical_generator.generate_medical_advice(
+                user_query=query,
+                retrieval_results=retrieval_results,
+                intention=intention
+            )
+            generation_time = (datetime.now() - generation_start).total_seconds()
+            
+            generated_advice = medical_advice_result.get('medical_advice', '')
+            
+            if not generated_advice:
+                result = {
+                    "query": query,
+                    "category": category,
+                    "search_query": search_query,
+                    "pipeline_success": False,
+                    "coverage_score": 0.0,
+                    "error": "No generated advice",
+                    "timestamp": datetime.now().isoformat()
+                }
+                
+                self.coverage_results.append(result)
+                print(f"   ❌ No generated advice for coverage analysis")
+                return result
+            
+            # Step 4: Calculate coverage
+            coverage_analysis = self.calculate_coverage_score(generated_advice, processed_results)
+            
+            result = {
+                "query": query,
+                "category": category,
+                "search_query": search_query,
+                "pipeline_success": True,
+                "retrieval_time": retrieval_time,
+                "generation_time": generation_time,
+                "retrieved_docs_count": len(processed_results),
+                "generated_advice_length": len(generated_advice),
+                "coverage_analysis": coverage_analysis,
+                "coverage_score": coverage_analysis['coverage_score'],
+                "meets_threshold": coverage_analysis['meets_threshold'],
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            # Store result
+            self.coverage_results.append(result)
+            
+            print(f"   ✅ Pipeline: Complete")
+            print(f"   📊 Coverage Score: {coverage_analysis['coverage_score']:.3f} ({coverage_analysis['coverage_percentage']:.1f}%)")
+            print(f"   📝 Keywords: {coverage_analysis['matched_keywords_count']}/{coverage_analysis['source_keywords_count']} matched")
+            print(f"   🎯 Threshold: {'✅ Met' if result['meets_threshold'] else '❌ Not Met'}")
+            print(f"   ⏱️ Times: Retrieval={retrieval_time:.2f}s, Generation={generation_time:.2f}s")
+            
+            return result
+            
+        except Exception as e:
+            error_result = {
+                "query": query,
+                "category": category,
+                "pipeline_success": False,
+                "coverage_score": 0.0,
+                "error": str(e),
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            self.coverage_results.append(error_result)
+            print(f"   ❌ Coverage evaluation failed: {e}")
+            
+            return error_result
+    
+    def _detect_query_intention(self, query: str) -> str:
+        """Simplified query intention detection (from app.py)"""
+        query_lower = query.lower()
+        
+        if any(word in query_lower for word in ['diagnos', 'differential', 'possible', 'causes']):
+            return 'diagnosis'
+        elif any(word in query_lower for word in ['treat', 'manage', 'therapy', 'intervention']):
+            return 'treatment'
+        else:
+            return 'mixed'
+    
+    def parse_queries_from_file(self, filepath: str) -> Dict[str, List[Dict]]:
+        """Parse queries from file with category labels"""
+        print(f"📁 Reading queries from file: {filepath}")
+        
+        try:
+            with open(filepath, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            # Parse queries with category labels
+            queries_by_category = {
+                "diagnosis": [],
+                "treatment": [], 
+                "mixed": []
+            }
+            
+            lines = content.strip().split('\n')
+            
+            for line in lines:
+                line = line.strip()
+                if not line:
+                    continue
+                
+                # Parse format: "1.diagnosis: query text"
+                match = re.match(r'^\d+\.(diagnosis|treatment|mixed/complicated|mixed):\s*(.+)', line, re.IGNORECASE)
+                if match:
+                    category_raw = match.group(1).lower()
+                    query_text = match.group(2).strip()
+                    
+                    # Normalize category name
+                    if category_raw in ['mixed/complicated', 'mixed']:
+                        category = 'mixed'
+                    else:
+                        category = category_raw
+                    
+                    if category in queries_by_category and len(query_text) > 15:
+                        queries_by_category[category].append({
+                            "text": query_text,
+                            "category": category
+                        })
+            
+            print(f"📋 Parsed queries by category:")
+            for category, category_queries in queries_by_category.items():
+                print(f"  {category.capitalize()}: {len(category_queries)} queries")
+            
+            return queries_by_category
+            
+        except Exception as e:
+            print(f"❌ Failed to read file: {e}")
+            return {"error": f"Failed to read file: {e}"}
+    
+    def calculate_coverage_statistics(self) -> Dict[str, Any]:
+        """Calculate coverage statistics by category"""
+        category_stats = {}
+        all_successful_results = []
+        
+        # Group results by category
+        results_by_category = {
+            "diagnosis": [],
+            "treatment": [],
+            "mixed": []
+        }
+        
+        for result in self.coverage_results:
+            category = result.get('category', 'unknown')
+            if category in results_by_category:
+                results_by_category[category].append(result)
+                if result.get('pipeline_success'):
+                    all_successful_results.append(result)
+        
+        # Calculate statistics for each category
+        for category, results in results_by_category.items():
+            successful_results = [r for r in results if r.get('pipeline_success')]
+            
+            if successful_results:
+                coverage_scores = [r['coverage_score'] for r in successful_results]
+                avg_coverage = sum(coverage_scores) / len(coverage_scores)
+                avg_retrieval_time = sum(r.get('retrieval_time', 0) for r in successful_results) / len(successful_results)
+                avg_generation_time = sum(r.get('generation_time', 0) for r in successful_results) / len(successful_results)
+                
+                category_stats[category] = {
+                    "average_coverage": avg_coverage,
+                    "max_coverage": max(coverage_scores),
+                    "min_coverage": min(coverage_scores),
+                    "successful_evaluations": len(successful_results),
+                    "total_queries": len(results),
+                    "success_rate": len(successful_results) / len(results),
+                    "average_retrieval_time": avg_retrieval_time,
+                    "average_generation_time": avg_generation_time,
+                    "meets_threshold": avg_coverage >= 0.6,
+                    "individual_coverage_scores": coverage_scores
+                }
+            else:
+                category_stats[category] = {
+                    "average_coverage": 0.0,
+                    "max_coverage": 0.0,
+                    "min_coverage": 0.0,
+                    "successful_evaluations": 0,
+                    "total_queries": len(results),
+                    "success_rate": 0.0,
+                    "average_retrieval_time": 0.0,
+                    "average_generation_time": 0.0,
+                    "meets_threshold": False,
+                    "individual_coverage_scores": []
+                }
+        
+        # Calculate overall statistics
+        if all_successful_results:
+            all_coverage_scores = [r['coverage_score'] for r in all_successful_results]
+            overall_stats = {
+                "average_coverage": sum(all_coverage_scores) / len(all_coverage_scores),
+                "max_coverage": max(all_coverage_scores),
+                "min_coverage": min(all_coverage_scores),
+                "successful_evaluations": len(all_successful_results),
+                "total_queries": len(self.coverage_results),
+                "success_rate": len(all_successful_results) / len(self.coverage_results),
+                "meets_threshold": (sum(all_coverage_scores) / len(all_coverage_scores)) >= 0.6,
+                "target_compliance": (sum(all_coverage_scores) / len(all_coverage_scores)) >= 0.6
+            }
+        else:
+            overall_stats = {
+                "average_coverage": 0.0,
+                "max_coverage": 0.0,
+                "min_coverage": 0.0,
+                "successful_evaluations": 0,
+                "total_queries": len(self.coverage_results),
+                "success_rate": 0.0,
+                "meets_threshold": False,
+                "target_compliance": False
+            }
+        
+        return {
+            "category_results": category_stats,
+            "overall_results": overall_stats,
+            "timestamp": datetime.now().isoformat()
+        }
+    
+    def save_coverage_statistics(self, filename: str = None) -> str:
+        """Save coverage statistics for chart generation"""
+        stats = self.calculate_coverage_statistics()
+        
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"coverage_statistics_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(stats, f, indent=2, ensure_ascii=False)
+        
+        print(f"📊 Coverage statistics saved to: {filepath}")
+        return str(filepath)
+    
+    def save_coverage_details(self, filename: str = None) -> str:
+        """Save detailed coverage results"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"coverage_details_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        # Create comprehensive coverage data
+        coverage_data = {
+            "evaluation_metadata": {
+                "total_queries": len(self.coverage_results),
+                "successful_evaluations": len([r for r in self.coverage_results if r.get('pipeline_success')]),
+                "timestamp": datetime.now().isoformat(),
+                "evaluator_type": "retrieval_coverage",
+                "threshold_used": 0.6
+            },
+            "coverage_results": self.coverage_results
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(coverage_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"📝 Coverage details saved to: {filepath}")
+        return str(filepath)
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent coverage evaluation interface"""
+    
+    print("📈 OnCall.ai Coverage Evaluator - Retrieval Coverage Analysis")
+    
+    if len(sys.argv) > 1:
+        query_file = sys.argv[1]
+    else:
+        # Default to evaluation/pre_user_query_evaluate.txt
+        query_file = Path(__file__).parent / "pre_user_query_evaluate.txt"
+    
+    if not os.path.exists(query_file):
+        print(f"❌ Query file not found: {query_file}")
+        print("Usage: python coverage_evaluator.py [query_file.txt]")
+        sys.exit(1)
+    
+    # Initialize evaluator
+    evaluator = CoverageEvaluator()
+    
+    # Parse queries from file
+    queries_by_category = evaluator.parse_queries_from_file(str(query_file))
+    
+    if "error" in queries_by_category:
+        print(f"❌ Failed to parse queries: {queries_by_category['error']}")
+        sys.exit(1)
+    
+    # Test coverage for each query (requires full pipeline)
+    print(f"\n🧪 Retrieval Coverage Testing (Full Pipeline Required)")
+    print(f"⚠️ Note: This evaluator requires LLM calls for advice generation")
+    
+    for category, queries in queries_by_category.items():
+        if not queries:
+            continue
+            
+        print(f"\n📂 Testing {category.upper()} coverage:")
+        
+        for i, query_info in enumerate(queries):
+            query_text = query_info['text']
+            
+            # Test coverage (requires full pipeline)
+            result = evaluator.evaluate_single_coverage(query_text, category)
+            
+            # Pause between queries to avoid rate limits
+            if i < len(queries) - 1:
+                print(f"   ⏳ Pausing 5s before next query...")
+                import time
+                time.sleep(5)
+        
+        # Longer pause between categories
+        if category != list(queries_by_category.keys())[-1]:
+            print(f"\n⏳ Pausing 10s before next category...")
+            import time
+            time.sleep(10)
+    
+    # Generate and save results
+    print(f"\n📊 Generating coverage analysis...")
+    
+    # Save statistics and details
+    stats_path = evaluator.save_coverage_statistics()
+    details_path = evaluator.save_coverage_details()
+    
+    # Print final summary
+    stats = evaluator.calculate_coverage_statistics()
+    category_results = stats['category_results']
+    overall_results = stats['overall_results']
+    
+    print(f"\n📊 === COVERAGE EVALUATION SUMMARY ===")
+    print(f"Overall Performance:")
+    print(f"   Average Coverage: {overall_results['average_coverage']:.3f} ({overall_results['average_coverage']*100:.1f}%)")
+    print(f"   Pipeline Success Rate: {overall_results['success_rate']:.1%}")
+    print(f"   60% Threshold: {'✅ Met' if overall_results['meets_threshold'] else '❌ Not Met'}")
+    
+    print(f"\nCategory Breakdown:")
+    for category, cat_stats in category_results.items():
+        if cat_stats['total_queries'] > 0:
+            print(f"   {category.capitalize()}: {cat_stats['average_coverage']:.3f} "
+                  f"({cat_stats['successful_evaluations']}/{cat_stats['total_queries']}) "
+                  f"[R:{cat_stats['average_retrieval_time']:.2f}s, G:{cat_stats['average_generation_time']:.2f}s]")
+    
+    print(f"\n✅ Coverage evaluation complete!")
+    print(f"📊 Statistics: {stats_path}")
+    print(f"📝 Details: {details_path}")

evaluation/{evaluation_instruction.md → old/evaluation_instruction.md}

@@ -1,4 +1,5 @@
 # Model use
+
 llm model: (for comparison) with our-own version.
 https://huggingface.co/aaditya/Llama3-OpenBioLLM-70B
 https://huggingface.co/m42-health/Llama3-Med42-70B
@@ -12,59 +13,59 @@ https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct
 """
 ```
 
-
 ### 評估執行流程
+
 ```python
 def run_complete_evaluation(model_name: str, test_cases: List[str]) -> Dict[str, Any]:
     """執行完整的六項指標評估"""
-    
+
     results = {
         "model": model_name,
         "metrics": {},
         "detailed_results": []
     }
-    
+
     total_latencies = []
     extraction_successes = []
     relevance_scores = []
     coverage_scores = []
     actionability_scores = []
     evidence_scores = []
-    
+
     for query in test_cases:
         # 運行模型並測量所有指標
         start_time = time.time()
-        
+
         # 1. 總處理時長
         latency_result = measure_total_latency(query)
         total_latencies.append(latency_result['total_latency'])
-        
+
         # 2. 條件抽取成功率
         extraction_result = evaluate_condition_extraction([query])
         extraction_successes.append(extraction_result['success_rate'])
-        
+
         # 3 & 4. 檢索相關性和覆蓋率（需要實際檢索結果）
         retrieval_results = get_retrieval_results(query)
         relevance_result = evaluate_retrieval_relevance(retrieval_results)
         relevance_scores.append(relevance_result['average_relevance'])
-        
+
         generated_advice = get_generated_advice(query, retrieval_results)
         coverage_result = evaluate_retrieval_coverage(generated_advice, retrieval_results)
         coverage_scores.append(coverage_result['coverage'])
-        
+
         # 5 & 6. LLM 評估（需要完整回應）
         response_data = {
             'query': query,
             'advice': generated_advice,
             'retrieval_results': retrieval_results
         }
-        
+
         actionability_result = evaluate_clinical_actionability([response_data])
         actionability_scores.append(actionability_result[0]['overall_score'])
-        
+
         evidence_result = evaluate_clinical_evidence([response_data])
         evidence_scores.append(evidence_result[0]['overall_score'])
-        
+
         # 記錄詳細結果
         results["detailed_results"].append({
             "query": query,
@@ -75,7 +76,7 @@ def run_complete_evaluation(model_name: str, test_cases: List[str]) -> Dict[str,
             "actionability": actionability_result[0],
             "evidence": evidence_result[0]
         })
-    
+
     # 計算平均指標
     results["metrics"] = {
         "average_latency": sum(total_latencies) / len(total_latencies),
@@ -85,7 +86,7 @@ def run_complete_evaluation(model_name: str, test_cases: List[str]) -> Dict[str,
         "average_actionability": sum(actionability_scores) / len(actionability_scores),
         "average_evidence_score": sum(evidence_scores) / len(evidence_scores)
     }
-    
+
     return results
 ```
 
@@ -94,41 +95,43 @@ def run_complete_evaluation(model_name: str, test_cases: List[str]) -> Dict[str,
 ## 📈 評估結果分析框架
 
 ### 統計分析
+
 ```python
 def analyze_evaluation_results(results_A: Dict, results_B: Dict, results_C: Dict) -> Dict:
     """比較三個模型的評估結果"""
-    
+
     models = ['Med42-70B_direct', 'RAG_enhanced', 'OpenBioLLM-70B']
     metrics = ['latency', 'extraction_success_rate', 'relevance', 'coverage', 'actionability', 'evidence_score']
-    
+
     comparison = {}
-    
+
     for metric in metrics:
         comparison[metric] = {
             models[0]: results_A['metrics'][f'average_{metric}'],
             models[1]: results_B['metrics'][f'average_{metric}'],
             models[2]: results_C['metrics'][f'average_{metric}']
         }
-        
+
         # 計算相對改進
         baseline = comparison[metric][models[0]]
         rag_improvement = ((comparison[metric][models[1]] - baseline) / baseline) * 100
-        
+
         comparison[metric]['rag_improvement_percent'] = rag_improvement
-    
+
     return comparison
 ```
 
 ### 報告生成
+
 ```python
 def generate_evaluation_report(comparison_results: Dict) -> str:
     """生成評估報告"""
-    
+
     report = f"""
     # OnCall.ai 系統評估報告
-    
+
     ## 評估摘要
-    
+
     | 指標 | Med42-70B | RAG增強版 | OpenBioLLM | RAG改進% |
     |------|-----------|-----------|------------|----------|
     | 處理時長 | {comparison_results['latency']['Med42-70B_direct']:.2f}s | {comparison_results['latency']['RAG_enhanced']:.2f}s | {comparison_results['latency']['OpenBioLLM-70B']:.2f}s | {comparison_results['latency']['rag_improvement_percent']:+.1f}% |
@@ -137,9 +140,9 @@ def generate_evaluation_report(comparison_results: Dict) -> str:
     | 檢索覆蓋率 | - | {comparison_results['coverage']['RAG_enhanced']:.1%} | - | - |
     | 臨床可操作性 | {comparison_results['actionability']['Med42-70B_direct']:.1f}/10 | {comparison_results['actionability']['RAG_enhanced']:.1f}/10 | {comparison_results['actionability']['OpenBioLLM-70B']:.1f}/10 | {comparison_results['actionability']['rag_improvement_percent']:+.1f}% |
     | 臨床證據評分 | {comparison_results['evidence_score']['Med42-70B_direct']:.1f}/10 | {comparison_results['evidence_score']['RAG_enhanced']:.1f}/10 | {comparison_results['evidence_score']['OpenBioLLM-70B']:.1f}/10 | {comparison_results['evidence_score']['rag_improvement_percent']:+.1f}% |
-    
+
     """
-    
+
     return report
 ```
 
@@ -148,6 +151,7 @@ def generate_evaluation_report(comparison_results: Dict) -> str:
 ## 🔧 實驗執行步驟
 
 ### 1. 環境準備
+
 ```bash
 # 設置 HuggingFace token（用於 Inference Providers）
 export HF_TOKEN=your_huggingface_token
@@ -157,48 +161,49 @@ export ONCALL_EVAL_MODE=true
 ```
 
 ### 2. 實驗執行腳本框架
+
 ```python
 # evaluation/run_evaluation.py
 def main():
     """主要評估執行函數"""
-    
+
     # 加載測試用例
     test_cases = MEDICAL_TEST_CASES
-    
+
     # 實驗 A: YanBo 系統評估
     print("🔬 開始實驗 A: YanBo 系統評估")
     results_med42_direct = run_complete_evaluation("Med42-70B_direct", test_cases)
-    results_general_rag = run_complete_evaluation("Med42-70B_general_RAG", test_cases)  
+    results_general_rag = run_complete_evaluation("Med42-70B_general_RAG", test_cases)
     results_openbio = run_complete_evaluation("OpenBioLLM-70B", test_cases)
-    
+
     # 分析和報告
     comparison_A = analyze_evaluation_results(results_med42_direct, results_general_rag, results_openbio)
     report_A = generate_evaluation_report(comparison_A)
-    
+
     # 保存結果
     save_results("evaluation/results/yanbo_evaluation.json", {
         "comparison": comparison_A,
         "detailed_results": [results_med42_direct, results_general_rag, results_openbio]
     })
-    
+
     print("✅ 實驗 A 完成，結果已保存")
-    
+
     # 實驗 B: Jeff 系統評估
     print("🔬 開始實驗 B: Jeff 系統評估")
     results_med42_direct_b = run_complete_evaluation("Med42-70B_direct", test_cases)
     results_customized_rag = run_complete_evaluation("Med42-70B_customized_RAG", test_cases)
     results_openbio_b = run_complete_evaluation("OpenBioLLM-70B", test_cases)
-    
+
     # 分析和報告
     comparison_B = analyze_evaluation_results(results_med42_direct_b, results_customized_rag, results_openbio_b)
     report_B = generate_evaluation_report(comparison_B)
-    
+
     # 保存結果
     save_results("evaluation/results/jeff_evaluation.json", {
         "comparison": comparison_B,
         "detailed_results": [results_med42_direct_b, results_customized_rag, results_openbio_b]
     })
-    
+
     print("✅ 實驗 B 完成，結果已保存")
 
 if __name__ == "__main__":
@@ -206,6 +211,7 @@ if __name__ == "__main__":
 ```
 
 ### 3. 預期評估時間
+
 ```
 總評估時間估算：
 ├── 每個查詢處理時間：~30秒（包含LLM評估）
@@ -219,10 +225,11 @@ if __name__ == "__main__":
 ## 📊 評估成功標準
 
 ### 系統性能目標
+
 ```
 ✅ 達標條件：
 1. 總處理時長 ≤ 30秒
-2. 條件抽取成功率 ≥ 80%  
+2. 條件抽取成功率 ≥ 80%
 3. 檢索相關性 ≥ 0.2
 4. 檢索覆蓋率 ≥ 60%
 5. 臨床可操作性 ≥ 7.0/10
@@ -234,6 +241,7 @@ if __name__ == "__main__":
 ```
 
 ### 比較分析重點
+
 ```
 重點分析維度：
 ├── RAG 對處理時間的影響（可能增加延遲）
@@ -247,6 +255,7 @@ if __name__ == "__main__":
 ## 🛠️ 實施建議
 
 ### 分階段實施
+
 ```
 階段1: 基礎指標實現（1-4項）
 ├── 利用現有 app.py 中的時間測量
@@ -268,6 +277,7 @@ if __name__ == "__main__":
 ```
 
 ### 實施注意事項
+
 ```
 ⚠️ 重要提醒：
 1. 所有評估代碼應獨立於現有系統，避免影響正常運行
@@ -280,3 +290,412 @@ if __name__ == "__main__":
 ---
 
 **評估指南完成。請根據此指南實施評估實驗。**
+
+## Phase 1: Initial Assessment
+
+### Step 1.1 - 分析您的說明
+
+我重新理解了您的意思！
+
+### Step 1.2 - 無不清楚的需求
+
+### Step 1.3 - 確認理解（中文）
+
+完全明白了！您的評估架構是：
+
+## 🎯 **評估指標的測試層級**
+
+### **單獨測試 Retrieval（指標 2, 3, 4）**
+
+```python
+# 只測試 RAG 系統，因為只有 RAG 系統有這些組件：
+retrieval_only_metrics = [
+    "條件抽取成功率",    # 只有您的系統有 user_prompt.py
+    "檢索相關性",        # 只有 RAG 系統有檢索結果
+    "檢索覆蓋率"         # 只有 RAG 系統有檢索→生成的映射
+]
+
+# 測試對象：
+- Med42-70B_general_RAG（您的系統）✅
+- Med42-70B_customized_RAG（Jeff系統）✅
+- Med42-70B_direct（無RAG）❌ 沒有檢索組件
+- OpenBioLLM-70B（無RAG）❌ 沒有檢索組件
+```
+
+### **三模型一起測試（指標 1, 5, 6）**
+
+```python
+# 所有模型都能測試的指標：
+universal_metrics = [
+    "總處理時長",        # 所有模型都有響應時間
+    "臨床可操作性",      # Llama3-70B 評估所有模型的輸��
+    "臨床證據評分"       # Llama3-70B 評估所有模型的輸出
+]
+
+# 測試對象：
+- Med42-70B_direct ✅
+- Med42-70B_general_RAG ✅
+- OpenBioLLM-70B ✅
+```
+
+## 📊 **分層評估策略**
+
+### **階段 1：RAG 系統內部評估**
+
+```python
+# 只測試有 RAG 的系統
+rag_systems = ["Med42-70B_general_RAG", "Med42-70B_customized_RAG"]
+
+# 測試 RAG 特有指標
+for system in rag_systems:
+    evaluate_retrieval_metrics(system)  # 指標 2, 3, 4
+```
+
+### **階段 2：全模型比較評估**
+
+```python
+# 測試所有三個模型
+all_systems = ["Med42-70B_direct", "Med42-70B_general_RAG", "OpenBioLLM-70B"]
+
+# 測試通用指標
+for system in all_systems:
+    evaluate_universal_metrics(system)  # 指標 1, 5, 6
+```
+
+### **階段 3：綜合分析**
+
+```python
+# 合併兩階段結果，生成完整報告
+combine_evaluation_results()
+```
+
+### Step 1.4 - 無視覺修改確認 ✅
+
+此為評估策略討論，無涉及代碼修改。
+
+**您的理解完全正確！RAG 特有的指標只能在 RAG 系統內部測試，而通用指標可以跨所有模型比較。這樣的分層評估策略非常合理！**
+
+---
+
+## 📊 第七個評估指標（YanBo系統特有）
+
+### 7. 多層級 Fallback 效率（早期攔截率）
+
+**定義：** 系統通過多層級 Fallback 機制在早期層級成功處理查詢的效率
+
+**測量位置：** `src/user_prompt.py` 的 `extract_condition_keywords` 多層級處理邏輯
+
+**計算公式：**
+```
+Early_Interception_Rate = (Level1_Success + Level2_Success) / Total_Queries
+
+其中：
+- Level1_Success = 在預定義映射中直接找到條件的查詢數
+- Level2_Success = 通過LLM抽取成功的查詢數  
+- Total_Queries = 測試查詢總數
+
+時間節省效果：
+Time_Savings = (Late_Avg_Time - Early_Avg_Time) / Late_Avg_Time
+
+早期攔截效率：
+Efficiency_Score = Early_Interception_Rate × (1 + Time_Savings)
+```
+
+**ASCII 流程圖：**
+```
+多層級 Fallback 效率示意圖：
+┌─────────────┐    ┌─────────────┐    ┌─────────────┐
+│ 用戶查詢    │───▶│ Level 1     │───▶│ 直接成功    │
+│ "胸痛診斷"  │    │ 預定義映射  │    │ 35% (快)    │
+└─────────────┘    └─────────────┘    └─────────────┘
+                           │
+                           ▼ (失敗)
+                   ┌─────────────┐    ┌─────────────┐
+                   │ Level 2     │───▶│ LLM抽取成功 │
+                   │ LLM 條件抽取│    │ 40% (中等)  │
+                   └─────────────┘    └─────────────┘
+                           │
+                           ▼ (失敗)
+                   ┌─────────────┐    ┌─────────────┐
+                   │ Level 3-5   │───▶│ 後備成功    │
+                   │ 後續層級    │    │ 20% (慢)    │
+                   └─────────────┘    └─────────────┘
+                           │
+                           ▼ (失敗)
+                   ┌─────────────┐
+                   │ 完全失敗    │
+                   │ 5% (錯誤)   │
+                   └─────────────┘
+
+早期攔截率 = (35% + 40%) = 75% ✅ 目標 > 70%
+```
+
+**實現框架：**
+```python
+# 基於 user_prompt.py 的多層級處理邏輯
+def evaluate_early_interception_efficiency(test_queries: List[str]) -> Dict[str, float]:
+    """評估早期攔截率 - YanBo系統核心優勢"""
+    
+    level1_success = 0  # Level 1: 預定義映射成功
+    level2_success = 0  # Level 2: LLM 抽取成功
+    later_success = 0   # Level 3-5: 後續層級成功
+    total_failures = 0  # 完全失敗
+    
+    early_times = []    # 早期成功的處理時間
+    late_times = []     # 後期成功的處理時間
+    
+    for query in test_queries:
+        # 追蹤每個查詢的成功層級和時間
+        success_level, processing_time = track_query_success_level(query)
+        
+        if success_level == 1:
+            level1_success += 1
+            early_times.append(processing_time)
+        elif success_level == 2:
+            level2_success += 1
+            early_times.append(processing_time)
+        elif success_level in [3, 4, 5]:
+            later_success += 1
+            late_times.append(processing_time)
+        else:
+            total_failures += 1
+    
+    total_queries = len(test_queries)
+    early_success_count = level1_success + level2_success
+    
+    # 計算時間節省效果
+    early_avg_time = sum(early_times) / len(early_times) if early_times else 0
+    late_avg_time = sum(late_times) / len(late_times) if late_times else 0
+    time_savings = (late_avg_time - early_avg_time) / late_avg_time if late_avg_time > 0 else 0
+    
+    # 綜合效率分數
+    early_interception_rate = early_success_count / total_queries
+    efficiency_score = early_interception_rate * (1 + time_savings)
+    
+    return {
+        # 核心指標
+        "early_interception_rate": early_interception_rate,  # 早期攔截率
+        "level1_success_rate": level1_success / total_queries,
+        "level2_success_rate": level2_success / total_queries,
+        
+        # 時間效率
+        "early_avg_time": early_avg_time,
+        "late_avg_time": late_avg_time,
+        "time_savings_rate": time_savings,
+        
+        # 系統健康度
+        "total_success_rate": (total_queries - total_failures) / total_queries,
+        "miss_rate": total_failures / total_queries,
+        
+        # 綜合效率
+        "overall_efficiency_score": efficiency_score,
+        
+        # 詳細分布
+        "success_distribution": {
+            "level1": level1_success,
+            "level2": level2_success, 
+            "later_levels": later_success,
+            "failures": total_failures
+        }
+    }
+
+def track_query_success_level(query: str) -> Tuple[int, float]:
+    """
+    追蹤查詢在哪個層級成功並記錄時間
+    
+    Args:
+        query: 測試查詢
+        
+    Returns:
+        Tuple of (success_level, processing_time)
+    """
+    start_time = time.time()
+    
+    # 模擬 user_prompt.py 的層級處理邏輯
+    try:
+        # Level 1: 檢查預定義映射
+        if check_predefined_mapping(query):
+            processing_time = time.time() - start_time
+            return (1, processing_time)
+        
+        # Level 2: LLM 條件抽取
+        llm_result = llm_client.analyze_medical_query(query)
+        if llm_result.get('extracted_condition'):
+            processing_time = time.time() - start_time
+            return (2, processing_time)
+        
+        # Level 3: 語義搜索
+        semantic_result = semantic_search_fallback(query)
+        if semantic_result:
+            processing_time = time.time() - start_time
+            return (3, processing_time)
+        
+        # Level 4: 醫學驗證
+        validation_result = validate_medical_query(query)
+        if not validation_result:  # 驗證通過
+            processing_time = time.time() - start_time
+            return (4, processing_time)
+        
+        # Level 5: 通用搜索
+        generic_result = generic_medical_search(query)
+        if generic_result:
+            processing_time = time.time() - start_time
+            return (5, processing_time)
+        
+        # 完全失敗
+        processing_time = time.time() - start_time
+        return (0, processing_time)
+        
+    except Exception as e:
+        processing_time = time.time() - start_time
+        return (0, processing_time)
+
+def check_predefined_mapping(query: str) -> bool:
+    """檢查查詢是否在預定義映射中"""
+    # 基於 medical_conditions.py 的 CONDITION_KEYWORD_MAPPING
+    from medical_conditions import CONDITION_KEYWORD_MAPPING
+    
+    query_lower = query.lower()
+    for condition, keywords in CONDITION_KEYWORD_MAPPING.items():
+        if any(keyword.lower() in query_lower for keyword in keywords):
+            return True
+    return False
+```
+
+**目標閾值：** 
+- 早期攔截率 ≥ 70%（前兩層解決）
+- 時間節省率 ≥ 60%（早期比後期快）
+- 總成功率 ≥ 95%（漏接率 < 5%）
+
+---
+
+## 🧪 更新的完整評估流程
+
+### 測試用例設計
+```python
+# 基於 readme.md 中的範例查詢設計測試集
+MEDICAL_TEST_CASES = [
+    # Level 1 預期成功（預定義映射）
+    "患者胸痛怎麼處理？",
+    "心肌梗死的診斷方法？",
+    
+    # Level 2 預期成功（LLM抽取）
+    "60歲男性，有高血壓病史，突發胸痛。可能的原因和評估方法？",
+    "30歲患者突發嚴重頭痛和頸部僵硬。鑑別診斷？", 
+    
+    # Level 3+ 預期成功（複雜查詢）
+    "患者急性呼吸困難和腿部水腫。應該考慮什麼？",
+    "20歲女性，無病史，突發癲癇。可能原因和完整處理流程？",
+    
+    # 邊界測試
+    "疑似急性出血性中風。下一步處理？"
+]
+```
+
+### 更新的評估執行流程
+```python
+def run_complete_evaluation(model_name: str, test_cases: List[str]) -> Dict[str, Any]:
+    """執行完整的七項指標評估"""
+    
+    results = {
+        "model": model_name,
+        "metrics": {},
+        "detailed_results": []
+    }
+    
+    total_latencies = []
+    extraction_successes = []
+    relevance_scores = []
+    coverage_scores = []
+    actionability_scores = []
+    evidence_scores = []
+    fallback_efficiency_scores = []  # 新增
+    
+    for query in test_cases:
+        # 運行模型並測量所有指標
+        
+        # 1. 總處理時長
+        latency_result = measure_total_latency(query)
+        total_latencies.append(latency_result['total_latency'])
+        
+        # 2. 條件抽取成功率
+        extraction_result = evaluate_condition_extraction([query])
+        extraction_successes.append(extraction_result['success_rate'])
+        
+        # 3 & 4. 檢索相關性和覆蓋率
+        retrieval_results = get_retrieval_results(query)
+        relevance_result = evaluate_retrieval_relevance(retrieval_results)
+        relevance_scores.append(relevance_result['average_relevance'])
+        
+        generated_advice = get_generated_advice(query, retrieval_results)
+        coverage_result = evaluate_retrieval_coverage(generated_advice, retrieval_results)
+        coverage_scores.append(coverage_result['coverage'])
+        
+        # 5 & 6. LLM 評估
+        response_data = {
+            'query': query,
+            'advice': generated_advice,
+            'retrieval_results': retrieval_results
+        }
+        
+        actionability_result = evaluate_clinical_actionability([response_data])
+        actionability_scores.append(actionability_result[0]['overall_score'])
+        
+        evidence_result = evaluate_clinical_evidence([response_data])
+        evidence_scores.append(evidence_result[0]['overall_score'])
+        
+        # 7. 多層級 Fallback 效率（新增）
+        if model_name == "Med42-70B_general_RAG":  # 只對YanBo系統測量
+            fallback_result = evaluate_early_interception_efficiency([query])
+            fallback_efficiency_scores.append(fallback_result['overall_efficiency_score'])
+        
+        # 記錄詳細結果...
+    
+    # 計算平均指標
+    results["metrics"] = {
+        "average_latency": sum(total_latencies) / len(total_latencies),
+        "extraction_success_rate": sum(extraction_successes) / len(extraction_successes),
+        "average_relevance": sum(relevance_scores) / len(relevance_scores),
+        "average_coverage": sum(coverage_scores) / len(coverage_scores),
+        "average_actionability": sum(actionability_scores) / len(actionability_scores),
+        "average_evidence_score": sum(evidence_scores) / len(evidence_scores),
+        # 新增指標（只對RAG系統有效）
+        "average_fallback_efficiency": sum(fallback_efficiency_scores) / len(fallback_efficiency_scores) if fallback_efficiency_scores else 0.0
+    }
+    
+    return results
+```
+
+---
+
+## 📊 更新的系統成功標準
+
+### 系統性能目標（七個指標）
+```
+✅ 達標條件：
+1. 總處理時長 ≤ 30秒
+2. 條件抽取成功率 ≥ 80%  
+3. 檢索相關性 ≥ 0.25（基於實際醫學數據）
+4. 檢索覆蓋率 ≥ 60%
+5. 臨床可操作性 ≥ 7.0/10
+6. 臨床證據評分 ≥ 7.5/10
+7. 早期攔截率 ≥ 70%（多層級 Fallback 效率）
+
+🎯 YanBo RAG 系統成功標準：
+- RAG增強版在 5-7 項指標上優於基線 Med42-70B
+- 早期攔截率體現多層級設計的優勢
+- 整體提升幅度 ≥ 15%
+```
+
+### YanBo 系統特有優勢分析
+```
+多層級 Fallback 優勢：
+├── 漏接防護：通過多層級降低失敗率至 < 5%
+├── 時間優化：70%+ 查詢在前兩層快速解決
+├── 系統穩定：即使某層級失敗，後續層級提供保障
+└── 智能分流：不同複雜度查詢自動分配到合適層級
+```
+
+---
+
+**第七個指標已添加完成，專注測量您的多層級 Fallback 系統的早期攔截效率和時間節省效果。**

evaluation/old/extraction_evaluator.py

@@ -0,0 +1,379 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Condition Extraction Evaluator (Metric 2)
+============================================================
+
+Evaluates condition extraction success rate from user_prompt.py
+Pure automatic evaluation based on extract_condition_keywords() results
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any
+from datetime import datetime
+from pathlib import Path
+import re
+
+# Add project path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import existing system components
+try:
+    from user_prompt import UserPromptProcessor
+    from retrieval import BasicRetrievalSystem
+    from llm_clients import llm_Med42_70BClient
+except ImportError as e:
+    print(f"❌ Import failed: {e}")
+    print("Please ensure running from project root directory")
+    sys.exit(1)
+
+
+class ExtractionEvaluator:
+    """Condition extraction success rate evaluator - pure automatic evaluation"""
+    
+    def __init__(self):
+        """Initialize system components for extraction testing"""
+        print("🔧 Initializing Extraction Evaluator...")
+        
+        # Initialize required components for extraction
+        self.llm_client = llm_Med42_70BClient()
+        self.retrieval_system = BasicRetrievalSystem()
+        self.user_prompt_processor = UserPromptProcessor(
+            llm_client=self.llm_client,
+            retrieval_system=self.retrieval_system
+        )
+        
+        # Results accumulation
+        self.extraction_results = []
+        
+        print("✅ Extraction Evaluator initialization complete")
+    
+    def evaluate_single_extraction(self, query: str, category: str = "unknown") -> Dict[str, Any]:
+        """
+        Evaluate condition extraction success for a single query
+        
+        Tests user_prompt.py extract_condition_keywords() method
+        
+        Args:
+            query: Medical query to test
+            category: Query category (diagnosis/treatment/mixed)
+        """
+        print(f"🔍 Testing extraction for: {query[:50]}...")
+        print(f"📋 Category: {category}")
+        
+        try:
+            # Call the actual extraction method from user_prompt.py
+            extraction_start = datetime.now()
+            condition_result = self.user_prompt_processor.extract_condition_keywords(query)
+            extraction_time = (datetime.now() - extraction_start).total_seconds()
+            
+            # Analyze extraction success
+            extracted_condition = condition_result.get('condition')
+            query_status = condition_result.get('query_status')
+            emergency_keywords = condition_result.get('emergency_keywords', [])
+            treatment_keywords = condition_result.get('treatment_keywords', [])
+            fallback_level = condition_result.get('fallback_level', 'unknown')
+            
+            # Define success criteria
+            is_successful = (
+                extracted_condition and 
+                extracted_condition.strip() and 
+                extracted_condition != "unknown" and
+                query_status not in ['invalid_query', 'non_medical']
+            )
+            
+            result = {
+                "query": query,
+                "category": category,
+                "extraction_success": is_successful,
+                "extraction_time": extraction_time,
+                "extracted_condition": extracted_condition,
+                "query_status": query_status,
+                "emergency_keywords": emergency_keywords,
+                "treatment_keywords": treatment_keywords,
+                "fallback_level": fallback_level,
+                "full_condition_result": condition_result,
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            # Store result
+            self.extraction_results.append(result)
+            
+            print(f"   ✅ Extraction: {'Success' if is_successful else 'Failed'}")
+            print(f"   📝 Condition: {extracted_condition}")
+            print(f"   🎯 Status: {query_status}")
+            print(f"   ⏱️ Time: {extraction_time:.3f}s")
+            print(f"   🔄 Fallback Level: {fallback_level}")
+            
+            return result
+            
+        except Exception as e:
+            error_result = {
+                "query": query,
+                "category": category,
+                "extraction_success": False,
+                "extraction_time": 0.0,
+                "error": str(e),
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            self.extraction_results.append(error_result)
+            print(f"   ❌ Extraction failed: {e}")
+            
+            return error_result
+    
+    def parse_queries_from_file(self, filepath: str) -> Dict[str, List[Dict]]:
+        """Parse queries from file with category labels"""
+        print(f"📁 Reading queries from file: {filepath}")
+        
+        try:
+            with open(filepath, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            # Parse queries with category labels
+            queries_by_category = {
+                "diagnosis": [],
+                "treatment": [], 
+                "mixed": []
+            }
+            
+            lines = content.strip().split('\n')
+            
+            for line in lines:
+                line = line.strip()
+                if not line:
+                    continue
+                
+                # Parse format: "1.diagnosis: query text"
+                match = re.match(r'^\d+\.(diagnosis|treatment|mixed/complicated|mixed):\s*(.+)', line, re.IGNORECASE)
+                if match:
+                    category_raw = match.group(1).lower()
+                    query_text = match.group(2).strip()
+                    
+                    # Normalize category name
+                    if category_raw in ['mixed/complicated', 'mixed']:
+                        category = 'mixed'
+                    else:
+                        category = category_raw
+                    
+                    if category in queries_by_category and len(query_text) > 15:
+                        queries_by_category[category].append({
+                            "text": query_text,
+                            "category": category
+                        })
+            
+            print(f"📋 Parsed queries by category:")
+            for category, category_queries in queries_by_category.items():
+                print(f"  {category.capitalize()}: {len(category_queries)} queries")
+            
+            return queries_by_category
+            
+        except Exception as e:
+            print(f"❌ Failed to read file: {e}")
+            return {"error": f"Failed to read file: {e}"}
+    
+    def calculate_extraction_statistics(self) -> Dict[str, Any]:
+        """Calculate extraction success statistics by category"""
+        category_stats = {}
+        all_results = []
+        
+        # Group results by category
+        results_by_category = {
+            "diagnosis": [],
+            "treatment": [],
+            "mixed": []
+        }
+        
+        for result in self.extraction_results:
+            category = result.get('category', 'unknown')
+            if category in results_by_category:
+                results_by_category[category].append(result)
+                all_results.append(result)
+        
+        # Calculate statistics for each category
+        for category, results in results_by_category.items():
+            if results:
+                successful = [r for r in results if r.get('extraction_success')]
+                success_rate = len(successful) / len(results)
+                avg_time = sum(r.get('extraction_time', 0) for r in results) / len(results)
+                
+                category_stats[category] = {
+                    "success_rate": success_rate,
+                    "successful_count": len(successful),
+                    "total_count": len(results),
+                    "average_extraction_time": avg_time,
+                    "fallback_levels": [r.get('fallback_level') for r in results]
+                }
+            else:
+                category_stats[category] = {
+                    "success_rate": 0.0,
+                    "successful_count": 0,
+                    "total_count": 0,
+                    "average_extraction_time": 0.0,
+                    "fallback_levels": []
+                }
+        
+        # Calculate overall statistics
+        if all_results:
+            overall_successful = [r for r in all_results if r.get('extraction_success')]
+            overall_stats = {
+                "success_rate": len(overall_successful) / len(all_results),
+                "successful_count": len(overall_successful),
+                "total_count": len(all_results),
+                "average_extraction_time": sum(r.get('extraction_time', 0) for r in all_results) / len(all_results),
+                "target_compliance": len(overall_successful) / len(all_results) >= 0.8
+            }
+        else:
+            overall_stats = {
+                "success_rate": 0.0,
+                "successful_count": 0,
+                "total_count": 0,
+                "average_extraction_time": 0.0,
+                "target_compliance": False
+            }
+        
+        return {
+            "category_results": category_stats,
+            "overall_results": overall_stats,
+            "timestamp": datetime.now().isoformat()
+        }
+    
+    def save_extraction_statistics(self, filename: str = None) -> str:
+        """Save extraction statistics for chart generation"""
+        stats = self.calculate_extraction_statistics()
+        
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"extraction_statistics_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(stats, f, indent=2, ensure_ascii=False)
+        
+        print(f"📊 Extraction statistics saved to: {filepath}")
+        return str(filepath)
+    
+    def save_extraction_details(self, filename: str = None) -> str:
+        """Save detailed extraction results"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"extraction_details_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        # Create comprehensive extraction data
+        extraction_data = {
+            "evaluation_metadata": {
+                "total_queries": len(self.extraction_results),
+                "timestamp": datetime.now().isoformat(),
+                "evaluator_type": "condition_extraction"
+            },
+            "extraction_results": self.extraction_results
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(extraction_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"📝 Extraction details saved to: {filepath}")
+        return str(filepath)
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent extraction evaluation interface"""
+    
+    print("🔍 OnCall.ai Extraction Evaluator - Condition Extraction Success Rate")
+    
+    if len(sys.argv) > 1:
+        query_file = sys.argv[1]
+    else:
+        # Default to evaluation/pre_user_query_evaluate.txt
+        query_file = Path(__file__).parent / "pre_user_query_evaluate.txt"
+    
+    if not os.path.exists(query_file):
+        print(f"❌ Query file not found: {query_file}")
+        print("Usage: python extraction_evaluator.py [query_file.txt]")
+        sys.exit(1)
+    
+    # Initialize evaluator
+    evaluator = ExtractionEvaluator()
+    
+    # Parse queries from file
+    queries_by_category = evaluator.parse_queries_from_file(str(query_file))
+    
+    if "error" in queries_by_category:
+        print(f"❌ Failed to parse queries: {queries_by_category['error']}")
+        sys.exit(1)
+    
+    # Test extraction for each query
+    print(f"\n🧪 Condition Extraction Testing")
+    
+    for category, queries in queries_by_category.items():
+        if not queries:
+            continue
+            
+        print(f"\n📂 Testing {category.upper()} extraction:")
+        
+        for i, query_info in enumerate(queries):
+            query_text = query_info['text']
+            
+            # Test extraction
+            result = evaluator.evaluate_single_extraction(query_text, category)
+            
+            # Pause between queries to avoid rate limits (if needed)
+            if i < len(queries) - 1:
+                print(f"   ⏳ Pausing 3s before next query...")
+                import time
+                time.sleep(3)
+        
+        # Pause between categories
+        if category != list(queries_by_category.keys())[-1]:
+            print(f"\n⏳ Pausing 5s before next category...")
+            import time
+            time.sleep(5)
+    
+    # Generate and save results
+    print(f"\n📊 Generating extraction analysis...")
+    
+    # Save statistics and details
+    stats_path = evaluator.save_extraction_statistics()
+    details_path = evaluator.save_extraction_details()
+    
+    # Print final summary
+    stats = evaluator.calculate_extraction_statistics()
+    category_results = stats['category_results']
+    overall_results = stats['overall_results']
+    
+    print(f"\n📊 === EXTRACTION EVALUATION SUMMARY ===")
+    print(f"Overall Performance:")
+    print(f"   Success Rate: {overall_results['success_rate']:.1%}")
+    print(f"   Successful Extractions: {overall_results['successful_count']}/{overall_results['total_count']}")
+    print(f"   Average Extraction Time: {overall_results['average_extraction_time']:.3f}s")
+    print(f"   80% Target Compliance: {'✅ Met' if overall_results['target_compliance'] else '❌ Not Met'}")
+    
+    print(f"\nCategory Breakdown:")
+    for category, cat_stats in category_results.items():
+        if cat_stats['total_count'] > 0:
+            print(f"   {category.capitalize()}: {cat_stats['success_rate']:.1%} "
+                  f"({cat_stats['successful_count']}/{cat_stats['total_count']}) "
+                  f"[{cat_stats['average_extraction_time']:.3f}s avg]")
+    
+    print(f"\n✅ Extraction evaluation complete!")
+    print(f"📊 Statistics: {stats_path}")
+    print(f"📝 Details: {details_path}")

evaluation/old/relevance_evaluator.py

@@ -0,0 +1,447 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Retrieval Relevance Evaluator (Metric 3)
+===========================================================
+
+Evaluates retrieval relevance using cosine similarity from retrieval.py
+Automatic evaluation based on existing similarity scores with optional LLM sampling
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any
+from datetime import datetime
+from pathlib import Path
+import re
+import numpy as np
+
+# Add project path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import existing system components
+try:
+    from user_prompt import UserPromptProcessor
+    from retrieval import BasicRetrievalSystem
+    from llm_clients import llm_Med42_70BClient
+except ImportError as e:
+    print(f"❌ Import failed: {e}")
+    print("Please ensure running from project root directory")
+    sys.exit(1)
+
+
+class RelevanceEvaluator:
+    """Retrieval relevance evaluator using cosine similarity - automatic evaluation"""
+    
+    def __init__(self):
+        """Initialize system components for relevance testing"""
+        print("🔧 Initializing Relevance Evaluator...")
+        
+        # Initialize required components
+        self.llm_client = llm_Med42_70BClient()
+        self.retrieval_system = BasicRetrievalSystem()
+        self.user_prompt_processor = UserPromptProcessor(
+            llm_client=self.llm_client,
+            retrieval_system=self.retrieval_system
+        )
+        
+        # Results accumulation
+        self.relevance_results = []
+        
+        print("✅ Relevance Evaluator initialization complete")
+    
+    def evaluate_single_relevance(self, query: str, category: str = "unknown") -> Dict[str, Any]:
+        """
+        Evaluate retrieval relevance for a single query
+        
+        Uses existing cosine similarity scores from retrieval.py
+        
+        Args:
+            query: Medical query to test
+            category: Query category (diagnosis/treatment/mixed)
+        """
+        print(f"🔍 Testing relevance for: {query[:50]}...")
+        print(f"📋 Category: {category}")
+        
+        try:
+            # Step 1: Extract condition for search query construction
+            condition_result = self.user_prompt_processor.extract_condition_keywords(query)
+            
+            # Step 2: Perform retrieval (same as latency_evaluator.py)
+            search_query = f"{condition_result.get('emergency_keywords', '')} {condition_result.get('treatment_keywords', '')}".strip()
+            if not search_query:
+                search_query = condition_result.get('condition', query)
+            
+            retrieval_start = datetime.now()
+            retrieval_results = self.retrieval_system.search(search_query, top_k=5)
+            retrieval_time = (datetime.now() - retrieval_start).total_seconds()
+            
+            # Step 3: Extract similarity scores from retrieval results
+            processed_results = retrieval_results.get('processed_results', [])
+            
+            if not processed_results:
+                result = {
+                    "query": query,
+                    "category": category,
+                    "search_query": search_query,
+                    "retrieval_success": False,
+                    "average_relevance": 0.0,
+                    "relevance_scores": [],
+                    "retrieved_count": 0,
+                    "retrieval_time": retrieval_time,
+                    "error": "No retrieval results",
+                    "timestamp": datetime.now().isoformat()
+                }
+                
+                self.relevance_results.append(result)
+                print(f"   ❌ No retrieval results found")
+                return result
+            
+            # Extract cosine similarity scores
+            similarity_scores = []
+            retrieval_details = []
+            
+            for i, doc_result in enumerate(processed_results):
+                # Get similarity score (may be stored as 'distance', 'similarity_score', or 'score')
+                similarity = (
+                    doc_result.get('distance', 0.0) or 
+                    doc_result.get('similarity_score', 0.0) or
+                    doc_result.get('score', 0.0)
+                )
+                
+                similarity_scores.append(similarity)
+                
+                retrieval_details.append({
+                    "doc_index": i,
+                    "similarity_score": similarity,
+                    "content_snippet": doc_result.get('content', '')[:100] + "...",
+                    "doc_type": doc_result.get('type', 'unknown'),
+                    "source": doc_result.get('source', 'unknown')
+                })
+            
+            # Calculate relevance metrics
+            average_relevance = sum(similarity_scores) / len(similarity_scores)
+            max_relevance = max(similarity_scores)
+            min_relevance = min(similarity_scores)
+            
+            # Count high-relevance results (threshold: 0.2 based on evaluation_instruction.md)
+            high_relevance_count = sum(1 for score in similarity_scores if score >= 0.2)
+            high_relevance_ratio = high_relevance_count / len(similarity_scores)
+            
+            result = {
+                "query": query,
+                "category": category,
+                "search_query": search_query,
+                "retrieval_success": True,
+                "average_relevance": average_relevance,
+                "max_relevance": max_relevance,
+                "min_relevance": min_relevance,
+                "relevance_scores": similarity_scores,
+                "high_relevance_count": high_relevance_count,
+                "high_relevance_ratio": high_relevance_ratio,
+                "retrieved_count": len(processed_results),
+                "retrieval_time": retrieval_time,
+                "retrieval_details": retrieval_details,
+                "meets_threshold": average_relevance >= 0.2,
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            # Store result
+            self.relevance_results.append(result)
+            
+            print(f"   ✅ Retrieval: {len(processed_results)} documents")
+            print(f"   📊 Average Relevance: {average_relevance:.3f}")
+            print(f"   📈 High Relevance (≥0.2): {high_relevance_count}/{len(processed_results)} ({high_relevance_ratio:.1%})")
+            print(f"   🎯 Threshold: {'✅ Met' if result['meets_threshold'] else '❌ Not Met'}")
+            print(f"   ⏱️ Retrieval Time: {retrieval_time:.3f}s")
+            
+            return result
+            
+        except Exception as e:
+            error_result = {
+                "query": query,
+                "category": category,
+                "retrieval_success": False,
+                "average_relevance": 0.0,
+                "error": str(e),
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            self.relevance_results.append(error_result)
+            print(f"   ❌ Relevance evaluation failed: {e}")
+            
+            return error_result
+    
+    def parse_queries_from_file(self, filepath: str) -> Dict[str, List[Dict]]:
+        """Parse queries from file with category labels"""
+        print(f"📁 Reading queries from file: {filepath}")
+        
+        try:
+            with open(filepath, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            # Parse queries with category labels
+            queries_by_category = {
+                "diagnosis": [],
+                "treatment": [], 
+                "mixed": []
+            }
+            
+            lines = content.strip().split('\n')
+            
+            for line in lines:
+                line = line.strip()
+                if not line:
+                    continue
+                
+                # Parse format: "1.diagnosis: query text"
+                match = re.match(r'^\d+\.(diagnosis|treatment|mixed/complicated|mixed):\s*(.+)', line, re.IGNORECASE)
+                if match:
+                    category_raw = match.group(1).lower()
+                    query_text = match.group(2).strip()
+                    
+                    # Normalize category name
+                    if category_raw in ['mixed/complicated', 'mixed']:
+                        category = 'mixed'
+                    else:
+                        category = category_raw
+                    
+                    if category in queries_by_category and len(query_text) > 15:
+                        queries_by_category[category].append({
+                            "text": query_text,
+                            "category": category
+                        })
+            
+            print(f"📋 Parsed queries by category:")
+            for category, category_queries in queries_by_category.items():
+                print(f"  {category.capitalize()}: {len(category_queries)} queries")
+            
+            return queries_by_category
+            
+        except Exception as e:
+            print(f"❌ Failed to read file: {e}")
+            return {"error": f"Failed to read file: {e}"}
+    
+    def calculate_relevance_statistics(self) -> Dict[str, Any]:
+        """Calculate relevance statistics by category"""
+        category_stats = {}
+        all_successful_results = []
+        
+        # Group results by category
+        results_by_category = {
+            "diagnosis": [],
+            "treatment": [],
+            "mixed": []
+        }
+        
+        for result in self.relevance_results:
+            category = result.get('category', 'unknown')
+            if category in results_by_category:
+                results_by_category[category].append(result)
+                if result.get('retrieval_success'):
+                    all_successful_results.append(result)
+        
+        # Calculate statistics for each category
+        for category, results in results_by_category.items():
+            successful_results = [r for r in results if r.get('retrieval_success')]
+            
+            if successful_results:
+                avg_relevance = sum(r['average_relevance'] for r in successful_results) / len(successful_results)
+                relevance_scores = [r['average_relevance'] for r in successful_results]
+                avg_retrieval_time = sum(r.get('retrieval_time', 0) for r in successful_results) / len(successful_results)
+                
+                category_stats[category] = {
+                    "average_relevance": avg_relevance,
+                    "max_relevance": max(relevance_scores),
+                    "min_relevance": min(relevance_scores),
+                    "successful_retrievals": len(successful_results),
+                    "total_queries": len(results),
+                    "success_rate": len(successful_results) / len(results),
+                    "average_retrieval_time": avg_retrieval_time,
+                    "meets_threshold": avg_relevance >= 0.2,
+                    "individual_relevance_scores": relevance_scores
+                }
+            else:
+                category_stats[category] = {
+                    "average_relevance": 0.0,
+                    "max_relevance": 0.0,
+                    "min_relevance": 0.0,
+                    "successful_retrievals": 0,
+                    "total_queries": len(results),
+                    "success_rate": 0.0,
+                    "average_retrieval_time": 0.0,
+                    "meets_threshold": False,
+                    "individual_relevance_scores": []
+                }
+        
+        # Calculate overall statistics
+        if all_successful_results:
+            all_relevance_scores = [r['average_relevance'] for r in all_successful_results]
+            overall_stats = {
+                "average_relevance": sum(all_relevance_scores) / len(all_relevance_scores),
+                "max_relevance": max(all_relevance_scores),
+                "min_relevance": min(all_relevance_scores),
+                "successful_retrievals": len(all_successful_results),
+                "total_queries": len(self.relevance_results),
+                "success_rate": len(all_successful_results) / len(self.relevance_results),
+                "meets_threshold": (sum(all_relevance_scores) / len(all_relevance_scores)) >= 0.2,
+                "target_compliance": (sum(all_relevance_scores) / len(all_relevance_scores)) >= 0.25
+            }
+        else:
+            overall_stats = {
+                "average_relevance": 0.0,
+                "max_relevance": 0.0,
+                "min_relevance": 0.0,
+                "successful_retrievals": 0,
+                "total_queries": len(self.relevance_results),
+                "success_rate": 0.0,
+                "meets_threshold": False,
+                "target_compliance": False
+            }
+        
+        return {
+            "category_results": category_stats,
+            "overall_results": overall_stats,
+            "timestamp": datetime.now().isoformat()
+        }
+    
+    def save_relevance_statistics(self, filename: str = None) -> str:
+        """Save relevance statistics for chart generation"""
+        stats = self.calculate_relevance_statistics()
+        
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"relevance_statistics_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(stats, f, indent=2, ensure_ascii=False)
+        
+        print(f"📊 Relevance statistics saved to: {filepath}")
+        return str(filepath)
+    
+    def save_relevance_details(self, filename: str = None) -> str:
+        """Save detailed relevance results"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"relevance_details_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        # Create comprehensive relevance data
+        relevance_data = {
+            "evaluation_metadata": {
+                "total_queries": len(self.relevance_results),
+                "successful_retrievals": len([r for r in self.relevance_results if r.get('retrieval_success')]),
+                "timestamp": datetime.now().isoformat(),
+                "evaluator_type": "retrieval_relevance",
+                "threshold_used": 0.2
+            },
+            "relevance_results": self.relevance_results
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(relevance_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"📝 Relevance details saved to: {filepath}")
+        return str(filepath)
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent relevance evaluation interface"""
+    
+    print("📊 OnCall.ai Relevance Evaluator - Retrieval Relevance Analysis")
+    
+    if len(sys.argv) > 1:
+        query_file = sys.argv[1]
+    else:
+        # Default to evaluation/pre_user_query_evaluate.txt
+        query_file = Path(__file__).parent / "pre_user_query_evaluate.txt"
+    
+    if not os.path.exists(query_file):
+        print(f"❌ Query file not found: {query_file}")
+        print("Usage: python relevance_evaluator.py [query_file.txt]")
+        sys.exit(1)
+    
+    # Initialize evaluator
+    evaluator = RelevanceEvaluator()
+    
+    # Parse queries from file
+    queries_by_category = evaluator.parse_queries_from_file(str(query_file))
+    
+    if "error" in queries_by_category:
+        print(f"❌ Failed to parse queries: {queries_by_category['error']}")
+        sys.exit(1)
+    
+    # Test relevance for each query
+    print(f"\n🧪 Retrieval Relevance Testing")
+    
+    for category, queries in queries_by_category.items():
+        if not queries:
+            continue
+            
+        print(f"\n📂 Testing {category.upper()} relevance:")
+        
+        for i, query_info in enumerate(queries):
+            query_text = query_info['text']
+            
+            # Test relevance
+            result = evaluator.evaluate_single_relevance(query_text, category)
+            
+            # Pause between queries to avoid rate limits
+            if i < len(queries) - 1:
+                print(f"   ⏳ Pausing 3s before next query...")
+                import time
+                time.sleep(3)
+        
+        # Pause between categories
+        if category != list(queries_by_category.keys())[-1]:
+            print(f"\n⏳ Pausing 5s before next category...")
+            import time
+            time.sleep(5)
+    
+    # Generate and save results
+    print(f"\n📊 Generating relevance analysis...")
+    
+    # Save statistics and details
+    stats_path = evaluator.save_relevance_statistics()
+    details_path = evaluator.save_relevance_details()
+    
+    # Print final summary
+    stats = evaluator.calculate_relevance_statistics()
+    category_results = stats['category_results']
+    overall_results = stats['overall_results']
+    
+    print(f"\n📊 === RELEVANCE EVALUATION SUMMARY ===")
+    print(f"Overall Performance:")
+    print(f"   Average Relevance: {overall_results['average_relevance']:.3f}")
+    print(f"   Retrieval Success Rate: {overall_results['success_rate']:.1%}")
+    print(f"   0.2 Threshold: {'✅ Met' if overall_results['meets_threshold'] else '❌ Not Met'}")
+    print(f"   0.25 Target: {'✅ Met' if overall_results['target_compliance'] else '❌ Not Met'}")
+    
+    print(f"\nCategory Breakdown:")
+    for category, cat_stats in category_results.items():
+        if cat_stats['total_queries'] > 0:
+            print(f"   {category.capitalize()}: {cat_stats['average_relevance']:.3f} "
+                  f"({cat_stats['successful_retrievals']}/{cat_stats['total_queries']}) "
+                  f"[{cat_stats['average_retrieval_time']:.3f}s avg]")
+    
+    print(f"\n✅ Relevance evaluation complete!")
+    print(f"📊 Statistics: {stats_path}")
+    print(f"📝 Details: {details_path}")

evaluation/pre_user_query_evaluate.txt

@@ -0,0 +1,5 @@
+1.diagnosis: 60-year-old patient with hypertension history, sudden chest pain. What are possible causes and how to assess?
+
+2.treatment: Suspected acute ischemic stroke. Tell me the next steps to take
+
+3.mixed/complicated: 20 y/f , porphyria, sudden seizure. What are possible causes and complete management workflow?

evaluation/single_test_query.txt

@@ -0,0 +1 @@
+1.diagnosis: 60-year-old patient with hypertension history, sudden chest pain. What are possible causes and how to assess?

evaluation/user_query.txt

@@ -17,18 +17,18 @@ Suspected acute ischemic stroke. Tell me the next steps to take
 
 ### 一、Diagnosis-Focused（診斷為主）
 
-1. I have a 68-year-old man with atrial fibrillation presenting with sudden slurred speech and right-sided weakness—what are the possible diagnoses, and how would you evaluate them?
-2. A 40-year-old woman reports fever, urinary frequency, and dysuria—what differential diagnoses should I consider, and which tests would you order?
-3. A 50-year-old patient has progressive dyspnea on exertion and orthopnea over two weeks—what are the likely causes, and what diagnostic steps should I take?
+1. I have a 68-year-old man with atrial fibrillation presenting with sudden slurred speech and right-sided weakness. what are the possible diagnoses, and how would you evaluate them?
+2. A 40-year-old woman reports fever, urinary frequency, and dysuria. what differential diagnoses should I consider, and which tests would you order?
+3. A 50-year-old patient has progressive dyspnea on exertion and orthopnea over two weeks. what are the likely causes, and what diagnostic steps should I take?
 
 ### 二、Treatment-Focused（治療為主）
 
-4. ECG shows a suspected acute STEMI—what immediate interventions should I initiate in the next five minutes?
-5. I have a patient diagnosed with bacterial meningitis—what empiric antibiotic regimen and supportive measures should I implement?
+4. ECG shows a suspected acute STEMI. what immediate interventions should I initiate in the next five minutes?
+5. I have a patient diagnosed with bacterial meningitis. What empiric antibiotic regimen and supportive measures should I implement?
 6. A patient is in septic shock with BP 80/50 mmHg and HR 120 bpm—what fluid resuscitation and vasopressor strategy would you recommend?
 
 ### 三、Mixed（診斷＋治療綜合）
 
 7. A 75-year-old diabetic presents with a non-healing foot ulcer and fever—what differential for osteomyelitis, diagnostic workup, and management plan do you suggest?
-8. A 60-year-old COPD patient has worsening dyspnea and hypercapnia on ABG—how would you confirm the diagnosis, and what is your stepwise treatment approach?
-9. A 28-year-old woman is experiencing postpartum hemorrhage—what are the possible causes, what immediate resuscitation steps should I take, and how would you proceed with definitive management?
+8. A 60-year-old COPD patient has worsening dyspnea and hypercapnia on ABG. How would you confirm the diagnosis, and what is your stepwise treatment approach?
+9. A 28-year-old woman is experiencing postpartum hemorrhage. what are the possible causes, what immediate resuscitation steps should I take, and how would you proceed with definitive management?

src/generation.py

@@ -30,7 +30,7 @@ logger = logging.getLogger(__name__)
 
 # Fallback Generation Configuration (Simplified Architecture)
 FALLBACK_TIMEOUTS = {
-    "primary": 30.0,        # Primary Med42-70B with full RAG context
+    "primary": 60.0,        # Primary Med42-70B increased timeout for stable evaluation
     "fallback_1": 1.0,      # RAG template generation (renamed from fallback_2)
     "fallback_2": 0.1       # Minimal template generation (instant)
 }
@@ -277,7 +277,7 @@ class MedicalAdviceGenerator:
             
             # Sort by relevance (distance) and take top 6
             all_chunks_sorted = sorted(all_chunks, key=lambda x: x.get("distance", 999))
-            selected_chunks = all_chunks_sorted[:6]
+            selected_chunks = all_chunks_sorted[:6]  # Limit to top 6 most relevant
             
             logger.info(f"Selected chunks by relevance (no intention): {len(selected_chunks)} total")
         
@@ -308,14 +308,14 @@ class MedicalAdviceGenerator:
                 # Special formatting for hospital-specific guidelines
                 source_label = "Hospital Protocol"
                 context_part = f"""
-[Guideline {i}] (Source: {source_label}, Relevance: {1-distance:.3f})
-📋 {chunk.get('matched', 'Hospital Document')}
-{chunk_text}
+                [Guideline {i}] (Source: {source_label}, Relevance: {1-distance:.3f})
+                📋 {chunk.get('matched', 'Hospital Document')}
+                {chunk_text}
                 """.strip()
             else:
                 context_part = f"""
-[Guideline {i}] (Source: {chunk_type.title()}, Relevance: {1-distance:.3f})
-{chunk_text}
+                [Guideline {i}] (Source: {chunk_type.title()}, Angular Distance: {distance:.3f})
+                {chunk_text}
                 """.strip()
             
             context_parts.append(context_part)

src/llm_clients.py

@@ -9,6 +9,8 @@ Date: 2025-07-29
 
 import logging
 import os
+import json
+import re
 from typing import Dict, Optional, Union, List
 from huggingface_hub import InferenceClient
 from dotenv import load_dotenv
@@ -68,6 +70,91 @@ class llm_Med42_70BClient:
             self.logger.error(f"Detailed Error: {repr(e)}")
             raise ValueError(f"Failed to initialize Medical LLM client: {str(e)}") from e
 
+    def fix_json_formatting(self, response_text: str) -> str:
+        """
+        Fix common JSON formatting errors
+        
+        Args:
+            response_text: Raw response text that may contain JSON errors
+            
+        Returns:
+            Fixed JSON string
+        """
+        # 1. Fix missing commas between key-value pairs
+        # Look for "value" "key" pattern and add comma
+        fixed = re.sub(r'"\s*\n\s*"', '",\n  "', response_text)
+        
+        # 2. Fix missing commas between values and keys
+        fixed = re.sub(r'"\s*(["\[])', '",\1', fixed)
+        
+        # 3. Remove trailing commas
+        fixed = re.sub(r',\s*}', '}', fixed)
+        fixed = re.sub(r',\s*]', ']', fixed)
+        
+        # 4. Ensure string values are properly quoted
+        fixed = re.sub(r':\s*([^",{}\[\]]+)\s*([,}])', r': "\1"\2', fixed)
+        
+        return fixed
+
+    def parse_medical_response(self, response_text: str) -> Dict:
+        """
+        Enhanced JSON parsing logic with error recovery
+        
+        Args:
+            response_text: Raw response text from Med42-70B
+            
+        Returns:
+            Parsed response dictionary
+        """
+        try:
+            return json.loads(response_text)
+        except json.JSONDecodeError as e:
+            self.logger.warning(f"Initial JSON parsing failed: {e}")
+            
+            # Attempt to fix common JSON errors
+            try:
+                fixed_response = self.fix_json_formatting(response_text)
+                self.logger.info("Attempting to parse fixed JSON")
+                return json.loads(fixed_response)
+            except json.JSONDecodeError as e2:
+                self.logger.error(f"Fixed JSON parsing also failed: {e2}")
+                
+                # Try to extract partial information
+                try:
+                    return self.extract_partial_medical_info(response_text)
+                except:
+                    # Final fallback format
+                    return {
+                        "extracted_condition": "parsing_error",
+                        "confidence": "0.0",
+                        "is_medical": True,
+                        "raw_response": response_text,
+                        "error": str(e)
+                    }
+
+    def extract_partial_medical_info(self, response_text: str) -> Dict:
+        """
+        Extract partial medical information from malformed response
+        
+        Args:
+            response_text: Malformed response text
+            
+        Returns:
+            Dictionary with extracted information
+        """
+        # Try to extract condition
+        condition_match = re.search(r'"extracted_condition":\s*"([^"]*)"', response_text)
+        confidence_match = re.search(r'"confidence":\s*"([^"]*)"', response_text)
+        medical_match = re.search(r'"is_medical":\s*(true|false)', response_text)
+        
+        return {
+            "extracted_condition": condition_match.group(1) if condition_match else "unknown",
+            "confidence": confidence_match.group(1) if confidence_match else "0.0",
+            "is_medical": medical_match.group(1) == "true" if medical_match else True,
+            "raw_response": response_text,
+            "parsing_method": "partial_extraction"
+        }
+
     def analyze_medical_query(
         self, 
         query: str, 
@@ -138,6 +225,13 @@ class llm_Med42_70BClient:
             self.logger.info(f"Raw LLM Response: {response_text}")
             self.logger.info(f"Query Latency: {latency:.4f} seconds")
             
+            # Direct text extraction - system prompt expects plain text response
+            # Since the system prompt instructs LLM to "Return ONLY the primary condition name",
+            # we should directly extract from text instead of attempting JSON parsing
+            extracted_condition = self._extract_condition(response_text)
+            confidence = '0.8'
+            self.logger.info(f"Extracted condition from text: {extracted_condition}")
+            
             # Detect abnormal response
             if self._is_abnormal_response(response_text):
                 self.logger.error(f"❌ Abnormal LLM response detected: {response_text[:50]}...")
@@ -149,15 +243,12 @@ class llm_Med42_70BClient:
                     'latency': latency
                 }
             
-            # Extract condition from response
-            extracted_condition = self._extract_condition(response_text)
-            
             # Log the extracted condition
             self.logger.info(f"Extracted Condition: {extracted_condition}")
             
             return {
                 'extracted_condition': extracted_condition,
-                'confidence': '0.8',
+                'confidence': confidence,
                 'raw_response': response_text,
                 'latency': latency  # Add latency to the return dictionary
             }
@@ -264,7 +355,7 @@ Focus on: conditions, symptoms, procedures, body systems."""
 
     def _extract_condition(self, response: str) -> str:
         """
-        Extract medical condition from model response.
+        Extract medical condition from model response with support for multiple formats.
         
         Args:
             response: Full model-generated text
@@ -272,18 +363,29 @@ Focus on: conditions, symptoms, procedures, body systems."""
         Returns:
             Extracted medical condition or empty string if non-medical
         """
+        from medical_conditions import CONDITION_KEYWORD_MAPPING
+        
         # Check if this is a rejection response first
         if self._is_rejection_response(response):
             return ""
         
-        from medical_conditions import CONDITION_KEYWORD_MAPPING
+        # Try CONDITION: format first (primary format for structured responses)
+        match = re.search(r"CONDITION:\s*(.+)", response, re.IGNORECASE)
+        if not match:
+            # Try Primary condition: format as fallback
+            match = re.search(r"Primary condition:\s*(.+)", response, re.IGNORECASE)
+        
+        if match:
+            value = match.group(1).strip()
+            if value.upper() not in ["NONE", "", "UNKNOWN"]:
+                return value
         
-        # Search in known medical conditions
+        # Final fallback to keyword mapping for backward compatibility
         for condition in CONDITION_KEYWORD_MAPPING.keys():
             if condition.lower() in response.lower():
                 return condition
         
-        return response.split('\n')[0].strip() or ""
+        return ""
     
     def _is_abnormal_response(self, response: str) -> bool:
         """
@@ -439,5 +541,136 @@ def main():
             'total_execution_time': total_execution_time
         }
 
+
+class llm_Llama3_70B_JudgeClient:
+    """
+    Llama3-70B client specifically for LLM judge evaluation.
+    Used for metrics 5-6 evaluation: Clinical Actionability & Evidence Quality.
+    """
+    
+    def __init__(
+        self, 
+        model_name: str = "meta-llama/Meta-Llama-3-70B-Instruct",
+        timeout: float = 60.0
+    ):
+        """
+        Initialize Llama3-70B judge client for evaluation tasks.
+        
+        Args:
+            model_name: Hugging Face model name for Llama3-70B
+            timeout: API call timeout duration (longer for judge evaluation)
+        
+        Note: This client is specifically designed for third-party evaluation,
+              not for medical advice generation.
+        """
+        self.logger = logging.getLogger(__name__)
+        self.timeout = timeout
+        self.model_name = model_name
+        
+        # Get Hugging Face token from environment
+        hf_token = os.getenv('HF_TOKEN')
+        if not hf_token:
+            self.logger.error("HF_TOKEN is missing from environment variables.")
+            raise ValueError(
+                "HF_TOKEN not found in environment variables. "
+                "Please set HF_TOKEN in your .env file or environment."
+            )
+        
+        # Initialize Hugging Face Inference Client for judge evaluation
+        try:
+            self.client = InferenceClient(
+                provider="auto",
+                api_key=hf_token,
+            )
+            self.logger.info(f"Llama3-70B judge client initialized with model: {model_name}")
+            self.logger.info("Judge LLM: Evaluation tool only. Not for medical advice generation.")
+            
+        except Exception as e:
+            self.logger.error(f"Failed to initialize Llama3-70B judge client: {e}")
+            raise
+    
+    def generate_completion(self, prompt: str) -> Dict[str, Union[str, float]]:
+        """
+        Generate completion using Llama3-70B for judge evaluation.
+        
+        Args:
+            prompt: Evaluation prompt for medical advice assessment
+            
+        Returns:
+            Dict containing response content and timing information
+        """
+        import time
+        
+        start_time = time.time()
+        
+        try:
+            self.logger.info(f"Calling Llama3-70B Judge with evaluation prompt ({len(prompt)} chars)")
+            
+            # Call Llama3-70B for judge evaluation
+            completion = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=[
+                    {
+                        "role": "user",
+                        "content": prompt
+                    }
+                ],
+                max_tokens=2048,  # Sufficient for evaluation responses
+                temperature=0.1,   # Low temperature for consistent evaluation
+            )
+            
+            # Extract response content
+            response_content = completion.choices[0].message.content
+            
+            end_time = time.time()
+            latency = end_time - start_time
+            
+            self.logger.info(f"Llama3-70B Judge Response: {response_content[:100]}...")
+            self.logger.info(f"Judge Evaluation Latency: {latency:.4f} seconds")
+            
+            return {
+                'content': response_content,
+                'latency': latency,
+                'model': self.model_name,
+                'timestamp': time.time()
+            }
+            
+        except Exception as e:
+            end_time = time.time()
+            error_latency = end_time - start_time
+            
+            self.logger.error(f"Llama3-70B judge evaluation failed: {e}")
+            self.logger.error(f"Error occurred after {error_latency:.4f} seconds")
+            
+            return {
+                'content': f"Judge evaluation error: {str(e)}",
+                'latency': error_latency,
+                'error': str(e),
+                'model': self.model_name,
+                'timestamp': time.time()
+            }
+    
+    def batch_evaluate(self, evaluation_prompt: str) -> Dict[str, Union[str, float]]:
+        """
+        Specialized method for batch evaluation of medical advice.
+        Alias for generate_completion with judge-specific logging.
+        
+        Args:
+            evaluation_prompt: Batch evaluation prompt containing multiple queries
+            
+        Returns:
+            Dict containing batch evaluation results and timing
+        """
+        self.logger.info("Starting batch judge evaluation...")
+        result = self.generate_completion(evaluation_prompt)
+        
+        if 'error' not in result:
+            self.logger.info(f"Batch evaluation completed successfully in {result['latency']:.2f}s")
+        else:
+            self.logger.error(f"Batch evaluation failed: {result.get('error', 'Unknown error')}")
+        
+        return result
+
+
 if __name__ == "__main__":
     main()

src/medical_conditions.py

@@ -63,6 +63,18 @@ CONDITION_KEYWORD_MAPPING: Dict[str, Dict[str, str]] = {
     "seizure disorder": {
         "emergency": "seizure|status epilepticus|postictal state",
         "treatment": "antiepileptic drugs|EEG monitoring|neurology consult"
+    },
+    "postpartum hemorrhage": {
+    "emergency": "postpartum hemorrhage|uterine atony|placental retention|vaginal laceration",
+    "treatment": "uterine massage|IV oxytocin infusion|blood transfusion|surgical intervention"
+    },
+    "bacterial meningitis": {
+    "emergency": "bacterial meningitis|fever|headache|neck stiffness|altered mental status|meningitis|meningeal signs",
+    "treatment": "empiric antibiotics|ceftriaxone|vancomycin|dexamethasone|lumbar puncture"
+    },
+    "anaphylaxis": {
+    "emergency": "anaphylaxis|allergic reaction|airway compromise|hypotension",
+    "treatment": "epinephrine|adrenaline|IV fluids|antihistamine|corticosteroids"
     }
 }
 

src/user_prompt.py

@@ -255,13 +255,15 @@ Return ONLY the specified format."""
                 timeout=12.0     # Single call timeout
             )
             
+            # Get both raw response and extracted condition
+            raw_response = llama_response.get('raw_response', '').strip()
             response_text = llama_response.get('extracted_condition', '').strip()
             logger.info(f"🤖 Combined L2+4 result: {response_text}")
             
-            # Parse structured response
-            medical_status = self._extract_field(response_text, 'MEDICAL')
-            condition_name = self._extract_field(response_text, 'CONDITION')
-            confidence = self._extract_field(response_text, 'CONFIDENCE')
+            # Parse structured response from raw LLM output (not extracted condition)
+            medical_status = self._extract_field(raw_response, 'MEDICAL')
+            condition_name = self._extract_field(raw_response, 'CONDITION')
+            confidence = self._extract_field(raw_response, 'CONFIDENCE')
             
             # Non-medical query detection
             if medical_status == 'NO':

united_requirements.txt

@@ -0,0 +1,141 @@
+accelerate==1.9.0
+aiofiles==24.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.12.14
+aiosignal==1.4.0
+aiosqlite==0.21.0
+annotated-types==0.7.0
+annoy==1.17.3
+anyio==4.9.0
+attrs==25.3.0
+banks==2.2.0
+beautifulsoup4==4.13.4
+Brotli==1.1.0
+certifi==2025.7.14
+cffi==1.17.1
+charset-normalizer==3.4.2
+click==8.2.1
+colorama==0.4.6
+contourpy==1.3.3
+cryptography==45.0.5
+cycler==0.12.1
+dataclasses-json==0.6.7
+datasets==4.0.0
+Deprecated==1.2.18
+dill==0.3.8
+dirtyjson==1.0.8
+distro==1.9.0
+easyocr==1.7.2
+fastapi==0.116.1
+ffmpy==0.6.1
+filelock==3.18.0
+filetype==1.2.0
+fonttools==4.59.0
+frozenlist==1.7.0
+fsspec==2023.12.2
+gradio==5.38.0
+gradio_client==1.11.0
+greenlet==3.2.3
+griffe==1.9.0
+groovy==0.1.2
+h11==0.16.0
+hf-xet==1.1.5
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.33.5
+idna==3.10
+imageio==2.37.0
+Jinja2==3.1.6
+jiter==0.10.0
+joblib==1.5.1
+kiwisolver==1.4.8
+lazy_loader==0.4
+llama-index-core==0.12.50
+llama-index-embeddings-huggingface==0.5.5
+llama-index-instrumentation==0.4.0
+llama-index-llms-huggingface==0.5.0
+llama-index-workflows==1.2.0
+lxml==6.0.0
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+marshmallow==3.26.1
+matplotlib==3.10.3
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.6.3
+multiprocess==0.70.16
+mypy_extensions==1.1.0
+nest-asyncio==1.6.0
+networkx==3.5
+ninja==1.11.1.4
+nltk==3.9.1
+numpy==2.2.6
+openai==1.97.0
+opencv-python-headless==4.12.0.88
+orjson==3.11.1
+packaging==25.0
+pandas==2.2.3
+pdfminer.six==20250506
+pdfplumber==0.11.7
+pillow==11.3.0
+platformdirs==4.3.8
+propcache==0.3.2
+psutil==7.0.0
+pyarrow==21.0.0
+pyclipper==1.3.0.post6
+pycparser==2.22
+pydantic==2.11.7
+pydantic_core==2.33.2
+pydub==0.25.1
+Pygments==2.19.2
+PyMuPDF==1.26.3
+pyparsing==3.2.3
+pypdf==5.8.0
+pypdfium2==4.30.0
+python-bidi==0.6.6
+python-dateutil==2.9.0.post0
+python-dotenv==1.1.1
+python-multipart==0.0.20
+pytz==2025.2
+PyYAML==6.0.2
+regex==2024.11.6
+requests==2.32.4
+rich==14.0.0
+ruff==0.12.4
+safehttpx==0.1.6
+safetensors==0.5.3
+scikit-image==0.25.2
+scikit-learn==1.7.1
+scipy==1.16.0
+seaborn==0.13.2
+semantic-version==2.10.0
+sentence-transformers==3.0.1
+shapely==2.1.1
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.7
+SQLAlchemy==2.0.42
+starlette==0.47.2
+sympy==1.14.0
+tenacity==9.1.2
+threadpoolctl==3.6.0
+tifffile==2025.6.11
+tiktoken==0.10.0
+tokenizers==0.21.2
+tomlkit==0.13.3
+torch==2.7.1
+torchvision==0.22.1
+tqdm==4.67.1
+transformers==4.53.2
+typer==0.16.0
+typing-inspect==0.9.0
+typing-inspection==0.4.1
+typing_extensions==4.14.1
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.35.0
+websockets==15.0.1
+wrapt==1.17.2
+xxhash==3.5.0
+yarl==1.20.1