Merge pull request #9 from YanBoChen0928/newbranchYB-rollback

app.py

@@ -136,11 +136,16 @@ class OnCallAIInterface:
             processing_steps.append(f"   ⏱️ Processing Time: {step1_time:.3f}s")
             
             # Handle non-medical queries
-            if condition_result.get('type') == 'invalid_query':
+            if condition_result.get('query_status') in ['invalid_query', 'non_medical']:
                 non_medical_msg = condition_result.get('message', 'This appears to be a non-medical query.')
                 processing_steps.append("   🚫 Query identified as non-medical")
                 return non_medical_msg, '\n'.join(processing_steps), "{}"
             
+            # Handle medical query with no specific condition
+            if condition_result.get('query_status') == 'medical_no_condition':
+                processing_steps.append("   ℹ️ Medical query confirmed, no specific condition extracted")
+                # Continue with standard processing
+            
             # STEP 2: User Confirmation (Auto-simulated)
             processing_steps.append("\n🤝 Step 2: User confirmation (auto-confirmed for demo)")
             confirmation = self.user_prompt_processor.handle_user_confirmation(condition_result)

dataset/keywords/special_terms_emergency.json

@@ -1,26 +1,31 @@
 {
-    "cardiac": {
-        "mi": ["mi", "m.i.", "myocardial infarction", "MI"],
-        "acs": ["acs", "ACS", "acute coronary syndrome"]
-    },
-    "respiratory": {
-        "ards": ["ards", "ARDS", "acute respiratory distress syndrome"],
-        "respiratory_failure": ["respiratory failure", "resp failure", "RF"]
-    },
-    "neurological": {
-        "loc": ["loc", "LOC", "loss of consciousness"],
-        "cva": ["cva", "CVA", "stroke", "cerebrovascular accident"]
-    },
-    "shock": {
-        "shock": ["shock", "circulatory failure"],
-        "septic_shock": ["septic shock", "sepsis induced shock"]
-    },
-    "bleeding": {
-        "gi_bleed": ["gi bleed", "gi bleeding", "gastrointestinal hemorrhage", "GI hemorrhage"],
-        "hemorrhage": ["hemorrhage", "bleeding", "blood loss"]
-    },
-    "vital_signs": {
-        "hypotension": ["hypotension", "low bp", "low blood pressure"],
-        "tachycardia": ["tachycardia", "elevated heart rate", "fast heart rate"]
-    }
-} 
+  "cardiac": {
+    "mi": ["mi", "m.i.", "myocardial infarction", "MI", "STEMI", "NSTEMI"],
+    "acs": ["acs", "ACS", "acute coronary syndrome"]
+  },
+  "respiratory": {
+    "ards": ["ards", "ARDS", "acute respiratory distress syndrome"],
+    "respiratory_failure": ["respiratory failure", "resp failure", "RF"]
+  },
+  "neurological": {
+    "loc": ["loc", "LOC", "loss of consciousness"],
+    "cva": ["cva", "CVA", "stroke", "cerebrovascular accident"]
+  },
+  "shock": {
+    "shock": ["shock", "circulatory failure"],
+    "septic_shock": ["septic shock", "sepsis induced shock"]
+  },
+  "bleeding": {
+    "gi_bleed": [
+      "gi bleed",
+      "gi bleeding",
+      "gastrointestinal hemorrhage",
+      "GI hemorrhage"
+    ],
+    "hemorrhage": ["hemorrhage", "bleeding", "blood loss"]
+  },
+  "vital_signs": {
+    "hypotension": ["hypotension", "low bp", "low blood pressure"],
+    "tachycardia": ["tachycardia", "elevated heart rate", "fast heart rate"]
+  }
+}

evaluation/evaluation_instruction.md

@@ -0,0 +1,282 @@
+# Model use
+llm model: (for comparison) with our-own version.
+https://huggingface.co/aaditya/Llama3-OpenBioLLM-70B
+https://huggingface.co/m42-health/Llama3-Med42-70B
+
+evaluation model:
+https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct
+
+```python
+"""
+參閱 user_query.txt
+"""
+```
+
+
+### 評估執行流程
+```python
+def run_complete_evaluation(model_name: str, test_cases: List[str]) -> Dict[str, Any]:
+    """執行完整的六項指標評估"""
+    
+    results = {
+        "model": model_name,
+        "metrics": {},
+        "detailed_results": []
+    }
+    
+    total_latencies = []
+    extraction_successes = []
+    relevance_scores = []
+    coverage_scores = []
+    actionability_scores = []
+    evidence_scores = []
+    
+    for query in test_cases:
+        # 運行模型並測量所有指標
+        start_time = time.time()
+        
+        # 1. 總處理時長
+        latency_result = measure_total_latency(query)
+        total_latencies.append(latency_result['total_latency'])
+        
+        # 2. 條件抽取成功率
+        extraction_result = evaluate_condition_extraction([query])
+        extraction_successes.append(extraction_result['success_rate'])
+        
+        # 3 & 4. 檢索相關性和覆蓋率（需要實際檢索結果）
+        retrieval_results = get_retrieval_results(query)
+        relevance_result = evaluate_retrieval_relevance(retrieval_results)
+        relevance_scores.append(relevance_result['average_relevance'])
+        
+        generated_advice = get_generated_advice(query, retrieval_results)
+        coverage_result = evaluate_retrieval_coverage(generated_advice, retrieval_results)
+        coverage_scores.append(coverage_result['coverage'])
+        
+        # 5 & 6. LLM 評估（需要完整回應）
+        response_data = {
+            'query': query,
+            'advice': generated_advice,
+            'retrieval_results': retrieval_results
+        }
+        
+        actionability_result = evaluate_clinical_actionability([response_data])
+        actionability_scores.append(actionability_result[0]['overall_score'])
+        
+        evidence_result = evaluate_clinical_evidence([response_data])
+        evidence_scores.append(evidence_result[0]['overall_score'])
+        
+        # 記錄詳細結果
+        results["detailed_results"].append({
+            "query": query,
+            "latency": latency_result,
+            "extraction": extraction_result,
+            "relevance": relevance_result,
+            "coverage": coverage_result,
+            "actionability": actionability_result[0],
+            "evidence": evidence_result[0]
+        })
+    
+    # 計算平均指標
+    results["metrics"] = {
+        "average_latency": sum(total_latencies) / len(total_latencies),
+        "extraction_success_rate": sum(extraction_successes) / len(extraction_successes),
+        "average_relevance": sum(relevance_scores) / len(relevance_scores),
+        "average_coverage": sum(coverage_scores) / len(coverage_scores),
+        "average_actionability": sum(actionability_scores) / len(actionability_scores),
+        "average_evidence_score": sum(evidence_scores) / len(evidence_scores)
+    }
+    
+    return results
+```
+
+---
+
+## 📈 評估結果分析框架
+
+### 統計分析
+```python
+def analyze_evaluation_results(results_A: Dict, results_B: Dict, results_C: Dict) -> Dict:
+    """比較三個模型的評估結果"""
+    
+    models = ['Med42-70B_direct', 'RAG_enhanced', 'OpenBioLLM-70B']
+    metrics = ['latency', 'extraction_success_rate', 'relevance', 'coverage', 'actionability', 'evidence_score']
+    
+    comparison = {}
+    
+    for metric in metrics:
+        comparison[metric] = {
+            models[0]: results_A['metrics'][f'average_{metric}'],
+            models[1]: results_B['metrics'][f'average_{metric}'],
+            models[2]: results_C['metrics'][f'average_{metric}']
+        }
+        
+        # 計算相對改進
+        baseline = comparison[metric][models[0]]
+        rag_improvement = ((comparison[metric][models[1]] - baseline) / baseline) * 100
+        
+        comparison[metric]['rag_improvement_percent'] = rag_improvement
+    
+    return comparison
+```
+
+### 報告生成
+```python
+def generate_evaluation_report(comparison_results: Dict) -> str:
+    """生成評估報告"""
+    
+    report = f"""
+    # OnCall.ai 系統評估報告
+    
+    ## 評估摘要
+    
+    | 指標 | Med42-70B | RAG增強版 | OpenBioLLM | RAG改進% |
+    |------|-----------|-----------|------------|----------|
+    | 處理時長 | {comparison_results['latency']['Med42-70B_direct']:.2f}s | {comparison_results['latency']['RAG_enhanced']:.2f}s | {comparison_results['latency']['OpenBioLLM-70B']:.2f}s | {comparison_results['latency']['rag_improvement_percent']:+.1f}% |
+    | 條件抽取成功率 | {comparison_results['extraction_success_rate']['Med42-70B_direct']:.1%} | {comparison_results['extraction_success_rate']['RAG_enhanced']:.1%} | {comparison_results['extraction_success_rate']['OpenBioLLM-70B']:.1%} | {comparison_results['extraction_success_rate']['rag_improvement_percent']:+.1f}% |
+    | 檢索相關性 | - | {comparison_results['relevance']['RAG_enhanced']:.3f} | - | - |
+    | 檢索覆蓋率 | - | {comparison_results['coverage']['RAG_enhanced']:.1%} | - | - |
+    | 臨床可操作性 | {comparison_results['actionability']['Med42-70B_direct']:.1f}/10 | {comparison_results['actionability']['RAG_enhanced']:.1f}/10 | {comparison_results['actionability']['OpenBioLLM-70B']:.1f}/10 | {comparison_results['actionability']['rag_improvement_percent']:+.1f}% |
+    | 臨床證據評分 | {comparison_results['evidence_score']['Med42-70B_direct']:.1f}/10 | {comparison_results['evidence_score']['RAG_enhanced']:.1f}/10 | {comparison_results['evidence_score']['OpenBioLLM-70B']:.1f}/10 | {comparison_results['evidence_score']['rag_improvement_percent']:+.1f}% |
+    
+    """
+    
+    return report
+```
+
+---
+
+## 🔧 實驗執行步驟
+
+### 1. 環境準備
+```bash
+# 設置 HuggingFace token（用於 Inference Providers）
+export HF_TOKEN=your_huggingface_token
+
+# 設置評估模式
+export ONCALL_EVAL_MODE=true
+```
+
+### 2. 實驗執行腳本框架
+```python
+# evaluation/run_evaluation.py
+def main():
+    """主要評估執行函數"""
+    
+    # 加載測試用例
+    test_cases = MEDICAL_TEST_CASES
+    
+    # 實驗 A: YanBo 系統評估
+    print("🔬 開始實驗 A: YanBo 系統評估")
+    results_med42_direct = run_complete_evaluation("Med42-70B_direct", test_cases)
+    results_general_rag = run_complete_evaluation("Med42-70B_general_RAG", test_cases)  
+    results_openbio = run_complete_evaluation("OpenBioLLM-70B", test_cases)
+    
+    # 分析和報告
+    comparison_A = analyze_evaluation_results(results_med42_direct, results_general_rag, results_openbio)
+    report_A = generate_evaluation_report(comparison_A)
+    
+    # 保存結果
+    save_results("evaluation/results/yanbo_evaluation.json", {
+        "comparison": comparison_A,
+        "detailed_results": [results_med42_direct, results_general_rag, results_openbio]
+    })
+    
+    print("✅ 實驗 A 完成，結果已保存")
+    
+    # 實驗 B: Jeff 系統評估
+    print("🔬 開始實驗 B: Jeff 系統評估")
+    results_med42_direct_b = run_complete_evaluation("Med42-70B_direct", test_cases)
+    results_customized_rag = run_complete_evaluation("Med42-70B_customized_RAG", test_cases)
+    results_openbio_b = run_complete_evaluation("OpenBioLLM-70B", test_cases)
+    
+    # 分析和報告
+    comparison_B = analyze_evaluation_results(results_med42_direct_b, results_customized_rag, results_openbio_b)
+    report_B = generate_evaluation_report(comparison_B)
+    
+    # 保存結果
+    save_results("evaluation/results/jeff_evaluation.json", {
+        "comparison": comparison_B,
+        "detailed_results": [results_med42_direct_b, results_customized_rag, results_openbio_b]
+    })
+    
+    print("✅ 實驗 B 完成，結果已保存")
+
+if __name__ == "__main__":
+    main()
+```
+
+### 3. 預期評估時間
+```
+總評估時間估算：
+├── 每個查詢處理時間：~30秒（包含LLM評估）
+├── 測試用例數量：7個
+├── 模型數量：3個
+└── 總時間：~10-15分鐘每個實驗
+```
+
+---
+
+## 📊 評估成功標準
+
+### 系統性能目標
+```
+✅ 達標條件：
+1. 總處理時長 ≤ 30秒
+2. 條件抽取成功率 ≥ 80%  
+3. 檢索相關性 ≥ 0.2
+4. 檢索覆蓋率 ≥ 60%
+5. 臨床可操作性 ≥ 7.0/10
+6. 臨床證據評分 ≥ 7.5/10
+
+🎯 RAG 系統成功標準：
+- RAG增強版在 4-6 項指標上優於基線 Med42-70B
+- 整體提升幅度 ≥ 10%
+```
+
+### 比較分析重點
+```
+重點分析維度：
+├── RAG 對處理時間的影響（可能增加延遲）
+├── RAG 對回答質量的提升（可操作性和證據品質）
+├── 不同 RAG 策略的效果差異（general vs customized）
+└── 與其他醫學模型的競爭力比較
+```
+
+---
+
+## 🛠️ 實施建議
+
+### 分階段實施
+```
+階段1: 基礎指標實現（1-4項）
+├── 利用現有 app.py 中的時間測量
+├── 擴展 user_prompt.py 的條件抽取評估
+├── 增強 retrieval.py 的相關性分析
+└── 實現 generation.py 的覆蓋率計算
+
+階段2: LLM評估實現（5-6項）
+├── 設置 HuggingFace Inference Providers
+├── 實現 Llama3-70B 評估客戶端
+├── 測試評估 prompts 的穩定性
+└── 建立評估結果解析邏輯
+
+階段3: 完整實驗執行
+├── 準備標準測試用例
+├── 執行 YanBo 系統評估（實驗A）
+├── 執行 Jeff 系統評估（實驗B）
+└── 生成比較分析報告
+```
+
+### 實施注意事項
+```
+⚠️ 重要提醒：
+1. 所有評估代碼應獨立於現有系統，避免影響正常運行
+2. LLM 評估可能不穩定，建議多次運行取平均值
+3. 注意 API 費用控制，特別是 Llama3-70B 調用
+4. 保存詳細的中間結果，便於調試和分析
+5. 測試用例應涵蓋���同複雜度和醫學領域
+```
+
+---
+
+**評估指南完成。請根據此指南實施評估實驗。**

evaluation/evaluation_instruction_customization.md

@@ -0,0 +1,721 @@
+_Chunk
+- Semantic_Similarity_Score = cosine_similarity(generated_text, chunk_text)
+- 使用BGE-Large-Medical模型计算语义相似度
+
+# Model
+llm model: (for comparison) with our-own version.
+https://huggingface.co/aaditya/Llama3-OpenBioLLM-70B
+https://huggingface.co/m42-health/Llama3-Med42-70B
+
+evaluation model:
+https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct
+
+```python
+"""
+參閱 user_query.txt
+"""
+```
+
+# **ASCII 流程图：**
+```
+定制化系统覆盖率计算流程：
+┌─────────────┐    ┌─────────────┐    ┌─────────────┐
+│ 两阶段检索  │───▶│ 生成的      │───▶│ 计算使用    │
+│ 8-15个块    │    │ 医学建议    │    │ 比例        │
+└─────────────┘    └─────────────┘    └─────────────┘
+      │                    │                    │
+      ▼                    ▼                    ▼
+┌─────────────┐    ┌─────────────┐    ┌─────────────┐
+│ 医院PDF     │    │ 医学术语    │    │ Coverage =  │
+│ 文档内容    │    │ 匹配分析    │    │ 10/15 = 67% │
+└─────────────┘    └─────────────┘    └─────────────┘
+```
+
+**实现框架：**
+```python
+# 基于定制化系统的文档块分析
+def evaluate_customization_coverage(generated_advice: str, retrieved_chunks: List[Dict]) -> Dict[str, float]:
+    """评估定制化检索覆盖率"""
+    
+    if not retrieved_chunks:
+        return {"coverage": 0.0, "used_chunks": 0, "total_chunks": 0}
+    
+    used_chunks = 0
+    chunk_usage_details = []
+    
+    # 使用BGE-Large-Medical模型计算语义相似度
+    embedding_model = SentenceTransformer("BAAI/bge-large-zh-v1.5")  # Jeff系统使用的模型
+    
+    for i, chunk in enumerate(retrieved_chunks):
+        chunk_text = chunk.get('chunk_text', '')
+        document_name = chunk.get('document', '')
+        similarity_score = chunk.get('score', 0.0)
+        
+        # 方法1: 医学术语匹配（基于医院文档特有术语）
+        medical_terms = extract_hospital_medical_terms(chunk_text)
+        term_matches = count_medical_term_matches(generated_advice, medical_terms)
+        term_match_score = term_matches / len(medical_terms) if medical_terms else 0
+        
+        # 方法2: 语义相似度（使用BGE-Large-Medical）
+        chunk_embedding = embedding_model.encode([chunk_text])[0]
+        advice_embedding = embedding_model.encode([generated_advice])[0]
+        semantic_score = cosine_similarity([chunk_embedding], [advice_embedding])[0][0]
+        
+        # 综合评分（考虑原始检索分数）
+        usage_score = max(term_match_score, semantic_score, similarity_score)
+        
+        # 阈值判断（使用率 > 0.25 视为已使用，适应医学领域特点）
+        is_used = usage_score > 0.25
+        if is_used:
+            used_chunks += 1
+            
+        chunk_usage_details.append({
+            "chunk_id": i,
+            "document": document_name,
+            "original_score": similarity_score,
+            "term_match_score": term_match_score,
+            "semantic_score": semantic_score,
+            "usage_score": usage_score,
+            "is_used": is_used
+        })
+    
+    coverage = used_chunks / len(retrieved_chunks)
+    
+    return {
+        "coverage": coverage,
+        "used_chunks": used_chunks,
+        "total_chunks": len(retrieved_chunks),
+        "chunk_details": chunk_usage_details,
+        "average_original_score": sum(chunk['original_score'] for chunk in chunk_usage_details) / len(chunk_usage_details)
+    }
+
+def extract_hospital_medical_terms(text: str) -> List[str]:
+    """提取医院文档特有的医学术语"""
+    # 基于医院临床指南的专业术语库
+    hospital_medical_terms = []
+    # 实现细节：结合医院特定术语和标准医学词汇
+    return hospital_medical_terms
+```
+
+**目标阈值：** ≥ 55%（考虑医院文档的专业性和复杂性）
+
+---
+
+### 5. 临床可操作性（Clinical Actionability）
+
+**定义：** 基于医院定制文档生成建议的临床实用性
+
+**测量位置：** 独立评估模块，使用 LLM 评估
+
+**评估者：** `meta-llama/Llama-3.3-70B-Instruct`
+
+**评估 Prompt：**
+```python
+CUSTOMIZATION_ACTIONABILITY_PROMPT = """
+你是一位在该医院工作了15年的资深主治医师和临床科室主任。你非常熟悉医院的临床指南、诊疗流程和设备资源情况。
+
+现在有一位年轻医师询问临床问题，系统基于医院的内部文档给出了建议。请你评估这个建议在本医院环境下的临床可操作性。
+
+【原始临床问题】
+{original_query}
+
+【基于医院文档的建议】
+{medical_advice}
+
+【引用的医院文档片段】
+{hospital_document_chunks}
+
+【评估标准】
+请从以下四个维度评估在医院环境下的临床可操作性（每项 1-10 分）：
+
+1. **医院资源匹配度 (Hospital Resource Compatibility)**
+   - 建议所需的设备、药物、检查是否在医院可获得？
+   - 是否符合医院的实际诊疗能力和资源配置？
+   - 评分：1-10 分
+
+2. **医院流程一致性 (Hospital Process Consistency)**  
+   - 建议是否符合医院的标准诊疗流程？
+   - 是否与医院的临床路径和指南一致？
+   - 评分：1-10 分
+
+3. **实施可行性 (Implementation Feasibility)**
+   - 建议是否可在医院当前条件下立即实施？
+   - 涉及的科室协作和流程是否现实可行？
+   - 评分：1-10 分
+
+4. **医院特色适应性 (Hospital-Specific Adaptation)**
+   - 建议是否体现了医院的专科优势和特色？
+   - 是否考虑了医院的患者群体特点？
+   - 评分：1-10 分
+
+【回答格式】
+请严格按照以下 JSON 格式回答：
+
+```json
+{
+  "resource_compatibility_score": <1-10整数>,
+  "process_consistency_score": <1-10整数>, 
+  "implementation_feasibility_score": <1-10整数>,
+  "hospital_adaptation_score": <1-10整数>,
+  "overall_actionability_score": <四项平均分，保留一位小数>,
+  "detailed_feedback": "<详细说明各项评分理由，特别指出建议与医院实际情况的匹配程度>"
+}
+```
+
+请记住：作为本医院的资深医师，你的评估应该基于医院的实际情况和资源能力。
+"""
+```
+
+**计算公式：**
+```
+Hospital_Actionability = (Resource_Compatibility + Process_Consistency + Implementation_Feasibility + Hospital_Adaptation) / 4
+
+医院定制化评分标准：
+- 9-10 分：完全适合本医院，可立即实施
+- 7-8 分：高度适合，需要少量调整
+- 5-6 分：基本适合，需要一定适配
+- 3-4 分：部分适合，需要显著修改
+- 1-2 分：不适合本医院环境
+```
+
+**实现框架：**
+```python
+def evaluate_customization_actionability(responses: List[Dict]) -> List[Dict]:
+    """评估医院定制化系统的临床可操作性"""
+    evaluator_client = HuggingFaceInferenceClient(
+        model="meta-llama/Llama-3.3-70B-Instruct",
+        provider="sambanova"
+    )
+    
+    actionability_results = []
+    
+    for response in responses:
+        # 格式化医院文档片段
+        hospital_chunks = format_hospital_document_chunks(response.get('retrieval_results', []))
+        
+        prompt = CUSTOMIZATION_ACTIONABILITY_PROMPT.format(
+            original_query=response['query'],
+            medical_advice=response['advice'],
+            hospital_document_chunks=hospital_chunks
+        )
+        
+        # 调用 Llama3-70B 评估
+        evaluation = evaluator_client.chat_completion(
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.1,
+            max_tokens=1000
+        )
+        
+        # 解析 JSON 响应
+        try:
+            scores = json.loads(evaluation.choices[0].message.content)
+            actionability_results.append({
+                "model": response['model'],
+                "actionability_scores": scores,
+                "overall_score": scores['overall_actionability_score']
+            })
+        except json.JSONDecodeError:
+            actionability_results.append({
+                "model": response['model'],
+                "error": "Failed to parse evaluation",
+                "overall_score": 0.0
+            })
+    
+    return actionability_results
+
+def format_hospital_document_chunks(retrieval_results: List[Dict]) -> str:
+    """格式化医院文档片段用于评估"""
+    if not retrieval_results:
+        return "未找到相关医院文档"
+    
+    chunks_text = ""
+    for i, result in enumerate(retrieval_results[:5], 1):
+        doc_name = result.get('document', '未知文档')
+        chunk_content = result.get('chunk_text', '')
+        similarity = result.get('score', 0.0)
+        
+        chunks_text += f"""
+        【医院文档 {i}】
+        文档来源：{doc_name}
+        相关性：{similarity:.3f}
+        内容：{chunk_content}
+        
+        """
+    
+    return chunks_text.strip()
+```
+
+**目标阈值：** ≥ 7.5 分
+
+---
+
+### 6. 临床证据评分（Clinical Evidence Score）
+
+**定义：** 基于医院文档的建议的证据可靠性和科学性
+
+**测量位置：** 独立评估模块，使用 LLM 评估
+
+**评估者：** `meta-llama/Llama-3.3-70B-Instruct`
+
+**评估 Prompt：**
+```python
+CUSTOMIZATION_EVIDENCE_PROMPT = """
+你是一位医院医务处的循证医学专家和临床指南审查委员。你负责审核和更新医院的各种临床指南和诊疗规范，对医院内部文档的证据质量有深入了解。
+
+现在需要你评估一个基于医院内部文档生成的医学建议的证据基础品质。
+
+【原始临床问题】  
+{original_query}
+
+【基于医院文档的建��】
+{medical_advice}
+
+【引用的医院文档内容】
+{hospital_document_sources}
+
+【评估标准】
+请从以下四个维度评估医院文档证据品质（每项 1-10 分）：
+
+1. **医院文档权威性 (Hospital Document Authority)**
+   - 引用的医院文档是否为权威临床指南？
+   - 文档版本是否为最新且有效的？  
+   - 评分：1-10 分
+
+2. **证据与建议一致性 (Evidence-Recommendation Consistency)**
+   - 提供的建议是否与医院文档内容完全一致？
+   - 是否存在与医院指南矛盾的声明？
+   - 评分：1-10 分
+
+3. **医院标准符合性 (Hospital Standard Compliance)**
+   - 建议是否符合医院的诊疗标准和质量要求？
+   - 是否体现了医院的诊疗规范和特色？
+   - 评分：1-10 分
+
+4. **文档引用准确性 (Document Citation Accuracy)**
+   - 是否准确引用和解读了医院文档内容？
+   - 是否避免了对文档内容的误解或扭曲？
+   - 评分：1-10 分
+
+【回答格式】
+请严格按照以下 JSON 格式回答：
+
+```json
+{
+  "document_authority_score": <1-10整数>,
+  "consistency_score": <1-10整数>,
+  "hospital_standard_score": <1-10整数>, 
+  "citation_accuracy_score": <1-10整数>,
+  "overall_evidence_score": <四项平均分，保留一位小数>,
+  "detailed_feedback": "<详细说明各项评分理由，特别关注医院文档的使用是否准确和恰当>"
+}
+```
+
+请记住：作为医院的循证医学专家，你的评估应该确保建议完全符合医院的文档标准和诊疗规范。
+"""
+```
+
+**计算公式：**
+```
+Hospital_Evidence_Score = (Document_Authority + Consistency + Hospital_Standard + Citation_Accuracy) / 4
+
+医院文档证据等级：
+- Level A: 医院权威指南和临床路径
+- Level B: 医院科室诊疗规范和SOP
+- Level C: 医院培训材料和临床经验总结
+- Level D: 外部指南的医院本地化版本
+```
+
+**实现框架：**
+```python
+def evaluate_customization_evidence(responses: List[Dict]) -> List[Dict]:
+    """评估医院定制化系统的证据品质"""
+    evaluator_client = HuggingFaceInferenceClient(
+        model="meta-llama/Llama-3.3-70B-Instruct", 
+        provider="sambanova"
+    )
+    
+    evidence_results = []
+    
+    for response in responses:
+        # 格式化医院文档来源
+        hospital_sources = format_hospital_document_sources(response.get('retrieval_results', []))
+        
+        prompt = CUSTOMIZATION_EVIDENCE_PROMPT.format(
+            original_query=response['query'],
+            medical_advice=response['advice'],
+            hospital_document_sources=hospital_sources
+        )
+        
+        # 调用 Llama3-70B 评估
+        evaluation = evaluator_client.chat_completion(
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.1,
+            max_tokens=1200
+        )
+        
+        # 解析评估结果
+        try:
+            scores = json.loads(evaluation.choices[0].message.content)
+            evidence_results.append({
+                "model": response['model'],
+                "evidence_scores": scores,
+                "overall_score": scores['overall_evidence_score']
+            })
+        except json.JSONDecodeError:
+            evidence_results.append({
+                "model": response['model'], 
+                "error": "Failed to parse evaluation",
+                "overall_score": 0.0
+            })
+    
+    return evidence_results
+
+def format_hospital_document_sources(retrieval_results: List[Dict]) -> str:
+    """格式化医院文档来源用于证据评估"""
+    if not retrieval_results:
+        return "未引用医院文档"
+    
+    sources_text = ""
+    for i, result in enumerate(retrieval_results[:5], 1):
+        doc_name = result.get('document', '未知文档.pdf')
+        chunk_content = result.get('chunk_text', '')
+        score = result.get('score', 0.0)
+        metadata = result.get('metadata', {})
+        
+        sources_text += f"""
+        【医院文档来源 {i}】
+        文档名称：{doc_name}
+        相关性：{score:.3f}
+        内容片段：{chunk_content}
+        元数据：{metadata}
+        
+        """
+    
+    return sources_text.strip()
+```
+
+**目标阈值：** ≥ 8.0 分（医院内部文档应有更高的证据可靠性）
+
+---
+
+## 🧪 完整评估流程
+
+### 测试用例设计（医院场景）
+```python
+# 基于医院定制化系统的典型查询
+HOSPITAL_TEST_CASES = [
+    # 急诊科常见问题
+    "患者胸痛应该如何诊断和处理？",
+    "急性心肌梗死的诊断标准是什么？",
+    "儿童发热的处理流程？",
+    
+    # 内科专业问题
+    "孕妇头晕的鉴别诊断？",
+    "老年患者跌倒的风险评估和预防？",
+    
+    # 外科相关问题
+    "急性阑尾炎的手术指征？",
+    "术前准备的标准流程？"
+]
+```
+
+### 评估执行流程
+```python
+def run_customization_evaluation(model_name: str, test_cases: List[str]) -> Dict[str, Any]:
+    """执行定制化系统的完整评估"""
+    
+    results = {
+        "model": model_name,
+        "metrics": {},
+        "detailed_results": []
+    }
+    
+    total_latencies = []
+    efficiency_scores = []
+    relevance_scores = []
+    coverage_scores = []
+    actionability_scores = []
+    evidence_scores = []
+    
+    for query in test_cases:
+        # 1. 总处理时长
+        latency_result = measure_customization_latency(query)
+        total_latencies.append(latency_result['total_latency'])
+        
+        # 2. 两阶段检索效率
+        efficiency_result = evaluate_two_stage_efficiency([query])
+        efficiency_scores.append(efficiency_result['overall_efficiency'])
+        
+        # 3. 检索相关性（实际数据）
+        retrieval_results = get_customization_retrieval_results(query)
+        relevance_result = evaluate_customization_relevance(retrieval_results)
+        relevance_scores.append(relevance_result['average_relevance'])
+        
+        # 4. 检索覆盖率
+        generated_advice = get_customization_generated_advice(query, retrieval_results)
+        coverage_result = evaluate_customization_coverage(generated_advice, retrieval_results)
+        coverage_scores.append(coverage_result['coverage'])
+        
+        # 5 & 6. LLM 评估（医院定制化版本）
+        response_data = {
+            'query': query,
+            'advice': generated_advice,
+            'retrieval_results': retrieval_results
+        }
+        
+        actionability_result = evaluate_customization_actionability([response_data])
+        actionability_scores.append(actionability_result[0]['overall_score'])
+        
+        evidence_result = evaluate_customization_evidence([response_data])
+        evidence_scores.append(evidence_result[0]['overall_score'])
+        
+        # 记录详细结果
+        results["detailed_results"].append({
+            "query": query,
+            "latency": latency_result,
+            "efficiency": efficiency_result,
+            "relevance": relevance_result,
+            "coverage": coverage_result,
+            "actionability": actionability_result[0],
+            "evidence": evidence_result[0]
+        })
+    
+    # 计算平均指标
+    results["metrics"] = {
+        "average_latency": sum(total_latencies) / len(total_latencies),
+        "average_efficiency": sum(efficiency_scores) / len(efficiency_scores),
+        "average_relevance": sum(relevance_scores) / len(relevance_scores),
+        "average_coverage": sum(coverage_scores) / len(coverage_scores),
+        "average_actionability": sum(actionability_scores) / len(actionability_scores),
+        "average_evidence_score": sum(evidence_scores) / len(evidence_scores)
+    }
+    
+    return results
+```
+
+---
+
+## 📈 评估结果分析框架
+
+### 定制化系统特有分析
+```python
+def analyze_customization_results(results_A: Dict, results_B: Dict, results_C: Dict) -> Dict:
+    """比较三个模型在医院定制化场景下的表现"""
+    
+    models = ['OpenBioLLM-70B_direct', 'Jeff_customization_RAG', 'Med42-70B_direct']
+    metrics = ['latency', 'efficiency', 'relevance', 'coverage', 'actionability', 'evidence_score']
+    
+    comparison = {}
+    
+    for metric in metrics:
+        comparison[metric] = {
+            models[0]: results_A['metrics'][f'average_{metric}'],
+            models[1]: results_B['metrics'][f'average_{metric}'],
+            models[2]: results_C['metrics'][f'average_{metric}']
+        }
+        
+        # 计算定制化系统的优势
+        baseline_openbio = comparison[metric][models[0]]
+        baseline_med42 = comparison[metric][models[2]]
+        customization_score = comparison[metric][models[1]]
+        
+        # 相对于两个基线模型的改进
+        improvement_vs_openbio = ((customization_score - baseline_openbio) / baseline_openbio) * 100
+        improvement_vs_med42 = ((customization_score - baseline_med42) / baseline_med42) * 100
+        
+        comparison[metric]['improvement_vs_openbio'] = improvement_vs_openbio
+        comparison[metric]['improvement_vs_med42'] = improvement_vs_med42
+    
+    return comparison
+```
+
+### 医院定制化报告生成
+```python
+def generate_customization_report(comparison_results: Dict) -> str:
+    """生成医院定制化系统评估报告"""
+    
+    report = f"""
+    # 医院定制化RAG系统评估报告
+    
+    ## 评估摘要
+    
+    | 指标 | OpenBioLLM | 定制化系统 | Med42-70B | vs OpenBio | vs Med42 |
+    |------|------------|------------|-----------|-----------|----------|
+    | 处理时长 | {comparison_results['latency']['OpenBioLLM-70B_direct']:.2f}s | {comparison_results['latency']['Jeff_customization_RAG']:.2f}s | {comparison_results['latency']['Med42-70B_direct']:.2f}s | {comparison_results['latency']['improvement_vs_openbio']:+.1f}% | {comparison_results['latency']['improvement_vs_med42']:+.1f}% |
+    | 检索效率 | - | {comparison_results['efficiency']['Jeff_customization_RAG']:.3f} | - | - | - |
+    | 检���相关性 | - | {comparison_results['relevance']['Jeff_customization_RAG']:.3f} | - | - | - |
+    | 检索覆盖率 | - | {comparison_results['coverage']['Jeff_customization_RAG']:.1%} | - | - | - |
+    | 临床可操作性 | {comparison_results['actionability']['OpenBioLLM-70B_direct']:.1f}/10 | {comparison_results['actionability']['Jeff_customization_RAG']:.1f}/10 | {comparison_results['actionability']['Med42-70B_direct']:.1f}/10 | {comparison_results['actionability']['improvement_vs_openbio']:+.1f}% | {comparison_results['actionability']['improvement_vs_med42']:+.1f}% |
+    | 临床证据评分 | {comparison_results['evidence_score']['OpenBioLLM-70B_direct']:.1f}/10 | {comparison_results['evidence_score']['Jeff_customization_RAG']:.1f}/10 | {comparison_results['evidence_score']['Med42-70B_direct']:.1f}/10 | {comparison_results['evidence_score']['improvement_vs_openbio']:+.1f}% | {comparison_results['evidence_score']['improvement_vs_med42']:+.1f}% |
+    
+    ## 定制化系统优势分析
+    
+    ### 🏥 医院特定性优势
+    - **文档本地化**: 基于医院实际临床指南和诊疗规范
+    - **资源匹配度**: 建议完全符合医院设备和人员配置
+    - **流程一致性**: 与医院现有诊疗流程高度契合
+    
+    ### ⚡ 技术架构优势  
+    - **两阶段检索**: 先文档筛选后块检索，精度和效率并重
+    - **BGE-Large-Medical**: 医学专用嵌入模型，语义理解更准确
+    - **Top-P智能过滤**: 动态质量阈值，避免低质量结果
+    
+    ### 📊 性能表现
+    - **检索速度**: 平均{comparison_results['latency']['Jeff_customization_RAG']:.1f}秒响应
+    - **检索精度**: {comparison_results['relevance']['Jeff_customization_RAG']:.1%}平均相关性
+    - **内容覆盖**: {comparison_results['coverage']['Jeff_customization_RAG']:.1%}文档内容利用率
+    """
+    
+    return report
+```
+
+---
+
+## 🔧 实验执行步骤
+
+### 1. 环境准备
+```bash
+# 设置 HuggingFace token（用于 Inference Providers）
+export HF_TOKEN=your_huggingface_token
+
+# 激活定制化系统环境
+source rag_env/bin/activate
+
+# 确保医院文档库已构建
+python customization/customization_pipeline.py
+```
+
+### 2. Jeff 系统评估脚本
+```python
+# evaluation/run_customization_evaluation.py
+def main():
+    """Jeff 定制化系统评估主函数"""
+    
+    # 加载医院场景测试用例
+    test_cases = HOSPITAL_TEST_CASES
+    
+    print("🏥 开始 Jeff 医院定制化系统评估")
+    
+    # 三个模型评估
+    results_openbio = run_customization_evaluation("OpenBioLLM-70B_direct", test_cases)
+    results_customization = run_customization_evaluation("Jeff_customization_RAG", test_cases)
+    results_med42 = run_customization_evaluation("Med42-70B_direct", test_cases)
+    
+    # 分析和报告
+    comparison_results = analyze_customization_results(results_openbio, results_customization, results_med42)
+    report = generate_customization_report(comparison_results)
+    
+    # 保存结果
+    save_results("evaluation/results/jeff_customization_evaluation.json", {
+        "comparison": comparison_results,
+        "detailed_results": [results_openbio, results_customization, results_med42],
+        "hospital_specific_analysis": analyze_hospital_advantages(results_customization)
+    })
+    
+    print("✅ Jeff 定制化系统评估完成，结果已保存")
+    
+    # 生成可视化图表
+    generate_customization_visualization(comparison_results)
+
+def analyze_hospital_advantages(customization_results: Dict) -> Dict:
+    """分析医院定制化系统的特有优势"""
+    return {
+        "document_diversity": len(set([r['document'] for r in customization_results['detailed_results']])),
+        "average_hospital_relevance": customization_results['metrics']['average_relevance'],
+        "two_stage_effectiveness": customization_results['metrics']['average_efficiency'],
+        "hospital_actionability": customization_results['metrics']['average_actionability']
+    }
+
+if __name__ == "__main__":
+    main()
+```
+
+### 3. 预期评估时间
+```
+医院定制化系统评估时间：
+├── 系统初始化：3-6秒（BGE-Large-Medical加载）
+├── 每个查询处理：1-3秒（两阶段检索）
+├── LLM评估：15-25秒/查询
+├── 测试用例数量：7个
+└── 总评估时间：8-12分钟
+```
+
+---
+
+## 📊 定制化系统成功标准
+
+### 系统性能目标（调整后）
+```
+✅ 达标条件：
+1. 总处理时长 ≤ 5秒
+2. 两阶段检索效率 ≥ 0.8  
+3. 检索相关性 ≥ 0.25（基于实际医学数据）
+4. 检索覆盖率 ≥ 55%
+5. 临床可操作性 ≥ 7.5/10（医院特定优势）
+6. 临床证据评分 ≥ 8.0/10（医院文档高可靠性）
+
+🎯 定制化系统成功标准：
+- 在医院特定场景下优于通用医学模型
+- 体现医院文档的本地化优势
+- 检索精度和速度的最佳平衡
+```
+
+### 关键性能指标（KPI）
+```
+医院定制化系统 KPI：
+├── 检索相关性改进���相比通用模型提升 15-25%
+├── 医院适用性：可操作性评分 > 7.5分
+├── 文档利用率：覆盖率 > 55%，体现定制化价值
+├── 响应速度：< 3秒平均响应时间
+└── 证据可靠性：> 8.0分，基于医院权威文档
+```
+
+---
+
+## 🛠️ 实施建议
+
+### 分阶段实施（Jeff 系统特定）
+```
+阶段1: 基础指标实现（1-4项）
+├── 利用现有 customization_pipeline.py 的时间测量
+├── 实现两阶段检索效率评估
+├── 分析 BGE-Large-Medical 的检索相关性
+└── 实现基于医院文档的覆盖率计算
+
+阶段2: LLM评估实现（5-6项）
+├── 设置 HuggingFace Inference Providers
+├── 实现 Llama3-70B 医院特定评估客户端
+├── 测试医院场景评估 prompts
+└── 建立医院文档引用分析逻辑
+
+阶段3: 完整实验执行
+├── 准备医院场景测试用例
+├── 执行 Jeff 定制化系统评估
+├── 对比通用医学模型表现
+└── 生成医院定制化优势分析报告
+```
+
+### 医院定制化特有注意事项
+```
+⚠️ 重要提醒：
+1. 确保医院文档库已正确构建和索引
+2. BGE-Large-Medical 模型需要充足内存（2-4GB）
+3. 两阶段检索的参数调优（top_p=0.6, min_similarity=0.3）
+4. 注意医院文档的隐私和安全要求
+5. 相关性阈值应基于实际医学检索数据（0.25-0.35）
+6. LLM 评估应强调医院特定性和本地化优势
+```
+
+### 与 YanBo 系统的区别
+```
+核心差异点：
+├── 数据源：医院内部文档 vs 公开医学指南
+├── 检索架构：两阶段 ANNOY vs 单阶段向量检索  
+├── 嵌入模型：BGE-Large-Medical vs PubMedBERT
+├── 优化目标：医院特定性 vs 通用医学知识
+└── 评估重点：本地化适应性 vs 标准医学质量
+```
+
+---
+
+**Jeff 医院定制化系统评估指南完成。请根据此指南实施定制化系统评估实验。**

evaluation/user_query.txt

@@ -0,0 +1,34 @@
+以下是九個以「我在問你」口吻設計的快速諮詢 prompts，分為三類，每類三題：
+
+
+1.
+Diagnosis-Focused 
+60-year-old patient with hypertension history, sudden chest pain. What are possible causes and how to assess?
+
+2.
+Treatment-Focused
+Suspected acute ischemic stroke. Tell me the next steps to take
+
+3.
+20 y/f , porphyria, sudden seizure. What are possible causes and complete management workflow?
+
+(測試時可以先用這三題看結果，如果要debug、調整完，再用下面的)
+---
+
+### 一、Diagnosis-Focused（診斷為主）
+
+1. I have a 68-year-old man with atrial fibrillation presenting with sudden slurred speech and right-sided weakness—what are the possible diagnoses, and how would you evaluate them?
+2. A 40-year-old woman reports fever, urinary frequency, and dysuria—what differential diagnoses should I consider, and which tests would you order?
+3. A 50-year-old patient has progressive dyspnea on exertion and orthopnea over two weeks—what are the likely causes, and what diagnostic steps should I take?
+
+### 二、Treatment-Focused（治療為主）
+
+4. ECG shows a suspected acute STEMI—what immediate interventions should I initiate in the next five minutes?
+5. I have a patient diagnosed with bacterial meningitis—what empiric antibiotic regimen and supportive measures should I implement?
+6. A patient is in septic shock with BP 80/50 mmHg and HR 120 bpm—what fluid resuscitation and vasopressor strategy would you recommend?
+
+### 三、Mixed（診斷＋治療綜合）
+
+7. A 75-year-old diabetic presents with a non-healing foot ulcer and fever—what differential for osteomyelitis, diagnostic workup, and management plan do you suggest?
+8. A 60-year-old COPD patient has worsening dyspnea and hypercapnia on ABG—how would you confirm the diagnosis, and what is your stepwise treatment approach?
+9. A 28-year-old woman is experiencing postpartum hemorrhage—what are the possible causes, what immediate resuscitation steps should I take, and how would you proceed with definitive management?

src/generation.py

@@ -28,22 +28,22 @@ logging.basicConfig(
 )
 logger = logging.getLogger(__name__)
 
-# Fallback Generation Configuration
+# Fallback Generation Configuration (Simplified Architecture)
 FALLBACK_TIMEOUTS = {
     "primary": 30.0,        # Primary Med42-70B with full RAG context
-    "fallback_1": 15.0,     # Simplified Med42-70B without RAG
-    "fallback_2": 1.0       # RAG template generation (instant)
+    "fallback_1": 1.0,      # RAG template generation (renamed from fallback_2)
+    "fallback_2": 0.1       # Minimal template generation (instant)
 }
 
 FALLBACK_TOKEN_LIMITS = {
-    "primary": 1200,         # Full comprehensive medical advice
-    "fallback_1": 600,      # Concise medical guidance
-    "fallback_2": 0         # Template-based, no LLM tokens
+    "primary": 1600,         # Full comprehensive medical advice (increased)
+    "fallback_1": 0,         # RAG template-based, no LLM tokens (renamed from fallback_2)
+    "fallback_2": 0          # Minimal template-based, no LLM tokens
 }
 
 FALLBACK_CONFIDENCE_SCORES = {
-    "fallback_1": 0.6,      # Med42-70B without RAG context
-    "fallback_2": 0.4       # RAG template only
+    "fallback_1": 0.4,       # RAG template only (renamed from fallback_2)
+    "fallback_2": 0.2        # Minimal template only
 }
 
 FALLBACK_ERROR_TRIGGERS = {
@@ -323,11 +323,12 @@ class MedicalAdviceGenerator:
         {focus_guidance}
 
         Provide guidance with:
+        • Prioritize information and evidence from above sources (PRIMARY)
+        • Use your medical knowledge to organize guidelines into actionable steps 
         • Numbered points (1. 2. 3.) for key steps
         • Line breaks between major sections
         • Highlight medications with dosages and routes
-        • Reference evidence from above sources
-        • Emphasize clinical judgment
+        • Emphasize clinical judgment for individual patient factors (SECONDARY)
 
         IMPORTANT: Keep response under 1000 words. Use concise numbered points. For complex cases with multiple conditions, address the most urgent condition first, then relevant comorbidities. Prioritize actionable clinical steps over theoretical explanations.
 
@@ -363,8 +364,9 @@ class MedicalAdviceGenerator:
             # Check for API errors in response
             if result.get('error'):
                 logger.warning(f"⚠️  Med42-70B returned error: {result['error']}")
-                # Attempt fallback instead of raising exception
-                return self._attempt_fallback_generation(prompt, result['error'])
+                # Pass any available content for potential simplification
+                primary_content = result.get('raw_response', '')
+                return self._attempt_fallback_generation(prompt, result['error'], primary_content)
             
             # Check for empty response
             if not result.get('raw_response', '').strip():
@@ -514,7 +516,7 @@ class MedicalAdviceGenerator:
             "disclaimer": "This system experienced a technical error. Please consult with qualified healthcare providers for medical decisions."
         }
 
-    def _attempt_fallback_generation(self, original_prompt: str, primary_error: str) -> Dict[str, Any]:
+    def _attempt_fallback_generation(self, original_prompt: str, primary_error: str, primary_result: str = None) -> Dict[str, Any]:
         """
         Orchestrate fallback generation attempts with detailed logging
         
@@ -524,21 +526,22 @@ class MedicalAdviceGenerator:
         Args:
             original_prompt: The complete RAG prompt that failed in primary generation
             primary_error: Error details from the primary generation attempt
+            primary_result: Primary result content (if available) for simplification
             
         Returns:
             Dict containing successful fallback response or final error response
         """
         logger.info("🔄 FALLBACK: Attempting fallback generation strategies")
         
-        # Fallback 1: Simplified Med42-70B without RAG context
+        # Fallback 1: RAG-only template response (renamed from fallback_2)
         try:
-            logger.info("📍 FALLBACK 1: Med42-70B without RAG context")
-            fallback_1_result = self._attempt_simplified_med42(original_prompt, primary_error)
+            logger.info("📍 FALLBACK 1: RAG-only template response")
+            fallback_1_result = self._attempt_rag_template(original_prompt, primary_error)
             
             if not fallback_1_result.get('error'):
-                logger.info("✅ FALLBACK 1: Success - Med42-70B without RAG")
-                # Mark response as fallback method 1
-                fallback_1_result['fallback_method'] = 'med42_simplified'
+                logger.info("✅ FALLBACK 1: Success - RAG template response")
+                # Mark response as fallback method 1 (renamed)
+                fallback_1_result['fallback_method'] = 'rag_template'
                 fallback_1_result['primary_error'] = primary_error
                 return fallback_1_result
             else:
@@ -547,20 +550,23 @@ class MedicalAdviceGenerator:
         except Exception as e:
             logger.error(f"❌ FALLBACK 1: Exception - {e}")
         
-        # Fallback 2: RAG-only template response
+        # Fallback 2: Minimal template response (renamed from fallback_3)
         try:
-            logger.info("📍 FALLBACK 2: RAG-only template response")
-            fallback_2_result = self._attempt_rag_template(original_prompt, primary_error)
-            
-            if not fallback_2_result.get('error'):
-                logger.info("✅ FALLBACK 2: Success - RAG template response")
-                # Mark response as fallback method 2
-                fallback_2_result['fallback_method'] = 'rag_template'
-                fallback_2_result['primary_error'] = primary_error
-                return fallback_2_result
-            else:
-                logger.warning(f"❌ FALLBACK 2: Failed - {fallback_2_result.get('error')}")
-                
+            logger.info("📍 FALLBACK 2: Minimal template response")
+            user_query = self._extract_user_query_from_prompt(original_prompt)
+            minimal_response = self._generate_minimal_template_response(user_query or "medical query")
+            
+            logger.info("✅ FALLBACK 2: Success - Minimal template response")
+            return {
+                'extracted_condition': 'minimal_template_response',
+                'confidence': str(FALLBACK_CONFIDENCE_SCORES['fallback_2']),
+                'raw_response': minimal_response,
+                'fallback_method': 'minimal_template',
+                'primary_error': primary_error,
+                'latency': 0.1,
+                'template_based': True
+            }
+            
         except Exception as e:
             logger.error(f"❌ FALLBACK 2: Exception - {e}")
         
@@ -588,15 +594,17 @@ class MedicalAdviceGenerator:
             'latency': 0.0
         }
 
-    def _attempt_simplified_med42(self, original_prompt: str, primary_error: str) -> Dict[str, Any]:
+
+    def _attempt_rag_template(self, original_prompt: str, primary_error: str) -> Dict[str, Any]:
         """
-        Attempt Med42-70B generation with simplified prompt (Fallback 1)
+        Generate template-based response using available RAG context (Fallback 1)
         
-        This method retries generation using the same Med42-70B model but with:
-        - Simplified prompt (user query only, no RAG context)
-        - Reduced timeout (15 seconds)
-        - Reduced token limit (300 tokens)
-        - Higher success probability due to reduced complexity
+        This method creates a structured response using retrieved medical guidelines
+        without LLM processing:
+        - Instant response (no API calls)
+        - Template-based formatting
+        - Uses extracted RAG context from original prompt
+        - Lower confidence score (0.4)
         
         Args:
             original_prompt: Original RAG prompt that failed
@@ -691,7 +699,7 @@ class MedicalAdviceGenerator:
         Returns:
             Dict with template response or error details
         """
-        logger.info("📍 FALLBACK 2: RAG-only template response")
+        logger.info("📍 FALLBACK 1: RAG-only template response")
         
         try:
             # Extract user query and RAG context from original prompt
@@ -699,7 +707,7 @@ class MedicalAdviceGenerator:
             rag_context = self._extract_rag_context_from_prompt(original_prompt)
             
             if not user_query:
-                logger.error("❌ FALLBACK 2: Failed to extract user query")
+                logger.error("❌ FALLBACK 1: Failed to extract user query")
                 return {
                     'error': 'Unable to extract user query for template response',
                     'fallback_method': 'rag_template'
@@ -713,11 +721,11 @@ class MedicalAdviceGenerator:
                 # Create full template response with RAG context
                 template_response = self._generate_rag_template_response(user_query, rag_context)
             
-            logger.info("✅ FALLBACK 2: Success - RAG template response")
+            logger.info("✅ FALLBACK 1: Success - RAG template response")
             
             return {
                 'extracted_condition': 'rag_template_response',
-                'confidence': str(FALLBACK_CONFIDENCE_SCORES['fallback_2']),  # 0.4
+                'confidence': str(FALLBACK_CONFIDENCE_SCORES['fallback_1']),  # 0.4 (renamed)
                 'raw_response': template_response,
                 'fallback_method': 'rag_template',
                 'primary_error': primary_error,

src/llm_clients.py

@@ -236,13 +236,7 @@ class llm_Med42_70BClient:
         
         # Detect short response (less than 2 characters)
         if len(response_stripped) < 2:
-            return True
-            
-        # Detect long response - allow some flexibility for detailed medical advice
-        # 750 words ≈ 1000-1200 chars, allow some flexibility to 2500 chars
-        if len(response_stripped) > 2500:  # Changed from 1000 to 2500
-            self.logger.warning(f"Response extremely long: {len(response_stripped)} chars")
-            return True
+            return True            
             
         return False
 

src/medical_conditions.py

@@ -55,6 +55,14 @@ CONDITION_KEYWORD_MAPPING: Dict[str, Dict[str, str]] = {
     "acute_coronary_syndrome": {
         "emergency": "ACS|chest pain|ECG changes",
         "treatment": "antiplatelet|statins|cardiac monitoring"
+    },
+    "acute seizure": {
+    "emergency": "seizure|convulsion|epilepsy|loss of consciousness",
+    "treatment": "anticonvulsant|benzodiazepine|neurologic assessment"
+    },
+    "seizure disorder": {
+        "emergency": "seizure|status epilepticus|postictal state",
+        "treatment": "antiepileptic drugs|EEG monitoring|neurology consult"
     }
 }
 

src/user_prompt.py

@@ -114,14 +114,35 @@ class UserPromptProcessor:
             return predefined_result
         logger.info("❌ LEVEL 1: FAILED - No predefined mapping found")
         
-        # Level 2: Llama3-Med42-70B Extraction (if available)
-        logger.info("📍 LEVEL 2: Attempting LLM extraction...")
+        # Level 2+4 Combined: Single LLM call for dual processing
+        logger.info("📍 LEVEL 2+4 COMBINED: Attempting unified extraction + validation")
+        if self.llm_client:
+            combined_result = self._combined_llm_extraction_validation(user_query)
+            if combined_result:
+                if combined_result['query_status'] == 'condition_found':
+                    logger.info("✅ LEVEL 2+4: SUCCESS - Condition extracted")
+                    return combined_result
+                elif combined_result['query_status'] in ['non_medical', 'invalid_query']:
+                    logger.info("✅ LEVEL 2+4: Query identified as non-medical")
+                    return combined_result
+                elif combined_result['query_status'] == 'medical_no_condition':
+                    logger.info("✅ LEVEL 2+4: Medical query confirmed, proceeding to semantic search")
+                    # Continue to Level 3 since query is validated as medical
+                else:
+                    logger.info("❌ LEVEL 2+4: Combined processing failed, falling back to individual levels")
+            else:
+                logger.info("❌ LEVEL 2+4: Combined processing failed, falling back to individual levels")
+        else:
+            logger.info("⏭️  LEVEL 2+4: SKIPPED - No LLM client available")
+
+        # Level 2: Fallback LLM Extraction (if combined failed)
+        logger.info("📍 LEVEL 2: Attempting individual LLM extraction...")
         if self.llm_client:
             llm_result = self._extract_with_llm(user_query)
             if llm_result:
-                logger.info("✅ LEVEL 2: SUCCESS - LLM extraction successful")
+                logger.info("✅ LEVEL 2: SUCCESS - Individual LLM extraction successful")
                 return llm_result
-            logger.info("❌ LEVEL 2: FAILED - LLM extraction failed")
+            logger.info("❌ LEVEL 2: FAILED - Individual LLM extraction failed")
         else:
             logger.info("⏭️  LEVEL 2: SKIPPED - No LLM client available")
         
@@ -153,9 +174,11 @@ class UserPromptProcessor:
         # No match found
         logger.warning("🚫 ALL LEVELS FAILED - Returning empty result")
         return {
+            'query_status': 'no_match_found',
             'condition': '',
             'emergency_keywords': '',
-            'treatment_keywords': ''
+            'treatment_keywords': '',
+            'message': 'Unable to process medical query'
         }
 
     def _predefined_mapping(self, user_query: str) -> Optional[Dict[str, str]]:
@@ -184,6 +207,127 @@ class UserPromptProcessor:
         
         return None
 
+    def _combined_llm_extraction_validation(self, user_query: str) -> Optional[Dict[str, Any]]:
+        """
+        Combined Level 2+4: Extract condition AND validate medical query in single LLM call
+        Expected time reduction: 16-25s → 12-15s (40% improvement)
+        
+        Args:
+            user_query: User's medical query
+            
+        Returns:
+            Dict with combined result indicating:
+            - condition_found: Specific medical condition extracted
+            - medical_no_condition: Medical query but no specific condition
+            - non_medical: Non-medical query (reject)
+        """
+        if not self.llm_client:
+            return None
+        
+        try:
+            # Combined prompt for both extraction and validation
+            combined_prompt = f"""Medical Query Analysis - Dual Task Processing:
+
+QUERY: "{user_query}"
+
+TASKS:
+1. Extract primary medical condition (if specific condition identifiable)
+2. Determine if this is a medical-related query
+
+RESPONSE FORMAT:
+MEDICAL: YES/NO
+CONDITION: [specific condition name or "NONE"]
+CONFIDENCE: [0.1-1.0]
+
+EXAMPLES:
+- "chest pain and shortness of breath" → MEDICAL: YES, CONDITION: Acute Coronary Syndrome, CONFIDENCE: 0.9
+- "how to cook pasta safely" → MEDICAL: NO, CONDITION: NONE, CONFIDENCE: 0.95  
+- "persistent headache treatment options" → MEDICAL: YES, CONDITION: Headache Disorder, CONFIDENCE: 0.8
+- "feeling unwell lately" → MEDICAL: YES, CONDITION: NONE, CONFIDENCE: 0.6
+
+Return ONLY the specified format."""
+
+            logger.info("🤖 COMBINED L2+4: Single LLM call for extraction + validation")
+            
+            llama_response = self.llm_client.analyze_medical_query(
+                query=combined_prompt,
+                max_tokens=100,  # Keep concise for condition name
+                timeout=12.0     # Single call timeout
+            )
+            
+            response_text = llama_response.get('extracted_condition', '').strip()
+            logger.info(f"🤖 Combined L2+4 result: {response_text}")
+            
+            # Parse structured response
+            medical_status = self._extract_field(response_text, 'MEDICAL')
+            condition_name = self._extract_field(response_text, 'CONDITION')
+            confidence = self._extract_field(response_text, 'CONFIDENCE')
+            
+            # Non-medical query detection
+            if medical_status == 'NO':
+                logger.info("✅ COMBINED L2+4: Identified as non-medical query")
+                return {
+                    'query_status': 'non_medical',
+                    'message': 'This appears to be a non-medical query.',
+                    'condition': '',
+                    'emergency_keywords': '',
+                    'treatment_keywords': '',
+                    'extraction_method': 'combined_llm_rejection',
+                    'confidence': float(confidence) if confidence else 0.9
+                }
+            
+            # Medical query with specific condition
+            if condition_name and condition_name != 'NONE':
+                standardized_condition = self._extract_condition_from_query(condition_name)
+                if standardized_condition:
+                    condition_details = get_condition_details(standardized_condition)
+                    if condition_details:
+                        logger.info(f"✅ COMBINED L2+4: Success - {standardized_condition}")
+                        return {
+                            'query_status': 'condition_found',
+                            'condition': standardized_condition,
+                            'emergency_keywords': condition_details['emergency'],
+                            'treatment_keywords': condition_details['treatment'],
+                            'extraction_method': 'combined_llm',
+                            'confidence': float(confidence) if confidence else 0.8
+                        }
+            
+            # Medical query but no specific condition extractable
+            if medical_status == 'YES':
+                logger.info("✅ COMBINED L2+4: Medical query confirmed, no specific condition")
+                return {
+                    'query_status': 'medical_no_condition',
+                    'message': 'Medical query confirmed, proceeding to semantic search',
+                    'condition': '',
+                    'emergency_keywords': '',
+                    'treatment_keywords': '',
+                    'extraction_method': 'combined_llm_medical_only',
+                    'confidence': float(confidence) if confidence else 0.6
+                }
+            
+            logger.info("❌ COMBINED L2+4: No valid condition extracted")
+            return None
+            
+        except Exception as e:
+            logger.error(f"Combined L2+4 extraction failed: {e}")
+            return None
+
+    def _extract_field(self, text: str, field_name: str) -> Optional[str]:
+        """
+        Extract specific field from structured LLM response
+        
+        Args:
+            text: Raw LLM response text
+            field_name: Field to extract (MEDICAL, CONDITION, CONFIDENCE)
+            
+        Returns:
+            Extracted field value or None
+        """
+        import re
+        pattern = rf"{field_name}:\s*([^\n,]+)"
+        match = re.search(pattern, text, re.IGNORECASE)
+        return match.group(1).strip() if match else None
+
     def _extract_with_llm(self, user_query: str) -> Optional[Dict[str, str]]:
         """
         Use Llama3-Med42-70B for advanced condition extraction
@@ -606,7 +750,7 @@ Please confirm:
             Dict with invalid query guidance
         """
         return {
-            'type': 'invalid_query',
+            'query_status': 'invalid_query',
             'message': "This is OnCall.AI, a clinical medical assistance platform. "
                        "Please input a medical problem you need help resolving. "
                        "\n\nExamples:\n"