Merge pull request #3 from YanBoChen0928/generation

Generation (complete user_input-> user_prompt (llm_client,medical_condition) ->retrieval.py->generation.py process)

.env.example

@@ -0,0 +1,2 @@
+# .env.example document
+HF_TOKEN=your_huggingface_token_here

.gitignore

@@ -1,17 +1,27 @@
 # 🧠 Virtual environments
 genAIvenv/
 .final_project_env/
+.env
+.venv
+env/
+venv/
 
 # 💻 OS / Editor garbage
 .DS_Store
 .vscode/
+*.swp
+*~
+.idea/
+*.iml
 
 # 📁 Documentation and project folders
 docs/
 dataset/dataset/
+cache/
 
 # 🧾 Compiled / output files
 *.pyc
+__pycache__/
 *.log
 *.zip
 *.tar.gz
@@ -20,6 +30,15 @@ dataset/dataset/
 *.json
 *.png
 
+# 🔑 Secrets and configs
+.env
+.env.local
+.env.*.local
+*.pem
+credentials.json
+token.json
+*.mdc
+
 # 🚫 Large files - models
 models/cache/
 models/cache/*.pkl
@@ -32,3 +51,15 @@ models/indices/annoy/*.ann
 *.pkl
 *.npy
 *.ann
+
+# 📊 Jupyter Notebook
+.ipynb_checkpoints
+*/.ipynb_checkpoints/*
+*.ipynb_checkpoints*
+
+# 📝 Coverage reports
+htmlcov/
+.coverage
+.coverage.*
+coverage.xml
+*.cover

requirements.txt

@@ -32,13 +32,16 @@ huggingface-hub==0.33.4
 idna==3.10
 Jinja2==3.1.6
 jiter==0.10.0
+joblib==1.5.1
 kiwisolver==1.4.8
 markdown-it-py==3.0.0
 MarkupSafe==3.0.2
 matplotlib==3.10.3
 mdurl==0.1.2
+mpmath==1.3.0
 multidict==6.6.3
 multiprocess==0.70.16
+networkx==3.5
 numpy==2.3.1
 openai==1.97.0
 orjson==3.11.0
@@ -53,6 +56,7 @@ pydub==0.25.1
 Pygments==2.19.2
 pyparsing==3.2.3
 python-dateutil==2.9.0.post0
+python-dotenv==1.1.1
 python-multipart==0.0.20
 pytz==2025.2
 PyYAML==6.0.2
@@ -62,6 +66,8 @@ rich==14.0.0
 ruff==0.12.4
 safehttpx==0.1.6
 safetensors==0.5.3
+scikit-learn==1.7.1
+scipy==1.16.1
 seaborn==0.13.2
 semantic-version==2.10.0
 sentence-transformers==3.0.1
@@ -69,8 +75,11 @@ shellingham==1.5.4
 six==1.17.0
 sniffio==1.3.1
 starlette==0.47.2
+sympy==1.14.0
+threadpoolctl==3.6.0
 tokenizers==0.21.2
 tomlkit==0.13.3
+torch==2.7.1
 tqdm==4.67.1
 transformers==4.53.2
 typer==0.16.0

src/Todo_20250731_Multlevel_Fallback_Fix.md

@@ -0,0 +1,83 @@
+太棒了！讓我們一起分析這個詳細的測試結果，這些數據很有價值！
+
+## 📊 **測試結果深度分析**
+
+### **✅ 成功的部分 (53.8% 成功率)**
+
+**Level 1 - 預定義映射**: 💯 **完美表現**
+- 3/3 測試通過，平均 0.000s（瞬間響應）
+- `acute myocardial infarction`, `acute stroke`, `pulmonary embolism` 都直接命中
+
+**Level 4b→5 - 冷門醫療查詢**: 💯 **完美表現**  
+- 3/3 測試通過，正確進入 generic search
+- 罕見血液疾病、ITP、壞死性筋膜炎都正確處理
+
+### **🔍 發現的關鍵問題**
+
+#### **問題1: Level 4 驗證機制失效** ❌
+**現象**: 非醫療查詢（烹飪、編程、天氣）都被當作醫療查詢處理
+```
+- "how to cook pasta properly?" → Level 5 (應該被拒絕)
+- "programming language" → Level 5 (應該被拒絕)  
+- "weather forecast" → Level 5 (應該被拒絕)
+```
+
+**根本原因**: `validate_medical_query` 邏輯有問題
+- LLM 雖然說"這不是醫療查詢"，但函數仍然返回 `None`（表示通過驗證）
+- 應該檢查 LLM 回應中是否明確說明"非醫療"
+
+#### **問題2: Level 3 語義搜索邏輯問題** ⚠️
+**現象**: 期望 Level 3 的查詢都跳到了 Level 5
+```
+- "emergency management of cardiovascular crisis" → Level 5 (期望 Level 3)
+- "urgent neurological intervention protocols" → Level 5 (期望 Level 3)
+```
+
+**原因**: `_infer_condition_from_text` 方法可能過於嚴格，無法推斷出有效條件
+
+#### **問題3: Level 2 行為不一致** ⚠️
+**現象**: 
+- `level2_001` 成功，但被 Level 1 攔截了（LLM 提取了已知條件）
+- `level2_002` 失敗，LLM 提取了條件但驗證失敗
+
+## 🛠️ **需要修正的優先順序**
+
+### **Priority 1: 修正 validate_medical_query**
+```python
+def validate_medical_query(self, user_query: str) -> Optional[Dict[str, Any]]:
+    # 檢查 LLM 回應是否明確說明非醫療
+    if llama_result.get('extracted_condition'):
+        response_text = llama_result.get('raw_response', '').lower()
+        
+        # 檢查是否明確拒絕醫療查詢
+        rejection_phrases = [
+            "not a medical condition", 
+            "outside my medical scope",
+            "unrelated to medical conditions",
+            "do not address"
+        ]
+        
+        if any(phrase in response_text for phrase in rejection_phrases):
+            return self._generate_invalid_query_response()
+        
+        return None  # 通過驗證
+```
+
+### **Priority 2: 改進語義搜索條件推斷**
+`_infer_condition_from_text` 的相似度閾值可能太高(0.7)，建議降低到 0.5
+
+### **Priority 3: 優化 Level 2 LLM 提取驗證**
+確保 `validate_condition` 能正確處理 LLM 的複雜回應
+
+## 🎯 **整體評估**
+
+### **速度表現**: ⭐⭐⭐⭐⭐
+- Level 1: 瞬間響應 (0.000s)
+- 平均: 14.4s（主要是 LLM 調用造成的）
+
+### **準確性**: ⭐⭐⭐
+- 預定義條件: 100% 準確
+- 冷門醫療: 100% 準確  
+- 非醫療拒絕: 0% 準確 ← **需要立即修正**
+
+你希望我先修正 `validate_medical_query` 的邏輯嗎？這是最關鍵的問題，解決後整體成功率應該能提升到 80%+。

src/__init__.py

@@ -3,6 +3,16 @@ OnCall.ai src package
 
 This package contains the core implementation of the OnCall.ai system.
 """
- 
+
 # Version
-__version__ = '0.1.0' 
+__version__ = '0.1.0'
+
+# import key modules
+from .llm_clients import llm_Med42_70BClient
+from .user_prompt import UserPromptProcessor
+from .retrieval import BasicRetrievalSystem
+from .medical_conditions import (
+    CONDITION_KEYWORD_MAPPING,
+    get_condition_keywords,
+    validate_condition
+) 

src/data_processing.py

@@ -106,7 +106,7 @@ class DataProcessor:
             raise FileNotFoundError(f"Treatment data not found: {treatment_path}")
         
         # Load data
-        self.emergency_data = pd.read_json(str(emergency_path), lines=True)  # 使用 str() 确保路径正确处理
+        self.emergency_data = pd.read_json(str(emergency_path), lines=True)  # use str() to ensure path is correct
         self.treatment_data = pd.read_json(str(treatment_path), lines=True)
         
         logger.info(f"Loaded {len(self.emergency_data)} emergency records")
@@ -167,11 +167,8 @@ class DataProcessor:
                 # Get the keyword text (already lowercase)
                 actual_keyword = text[keyword_pos:keyword_pos + len(keyword)]
                 
-                # Calculate rough window size using dynamic ratio
-                # Cap the rough chunk target token size to prevent tokenizer warnings
-                # Use 512 tokens as target (model's max limit)
-                ROUGH_CHUNK_TARGET_TOKENS = 512
-                char_window = int(ROUGH_CHUNK_TARGET_TOKENS * chars_per_token / 2)
+                # Calculate rough window size using simple ratio
+                char_window = int(chunk_size * chars_per_token / 2)
                 
                 # Get rough chunk boundaries in characters
                 rough_start = max(0, keyword_pos - char_window)
@@ -231,73 +228,119 @@ class DataProcessor:
         return chunks
     
     def create_dual_keyword_chunks(self, text: str, emergency_keywords: str, 
-                                 treatment_keywords: str, chunk_size: int = 512, 
+                                 treatment_keywords: str, chunk_size: int = None, 
                                  doc_id: str = None) -> List[Dict[str, Any]]:
         """
         Create chunks for treatment data with both emergency and treatment keywords
+        using token-based separate chunking strategy with enhanced metadata for treatment chunks
         
         Args:
             text: Input text
-            emergency_keywords: Emergency keywords
-            treatment_keywords: Treatment keywords
-            chunk_size: Size of each chunk
+            emergency_keywords: Emergency keywords (pipe-separated)
+            treatment_keywords: Treatment keywords (pipe-separated)
+            chunk_size: Size of each chunk in tokens (defaults to self.chunk_size)
             doc_id: Document ID for tracking
             
         Returns:
-            List of chunk dictionaries
+            List of chunk dictionaries with enhanced metadata for treatment chunks
         """
-        if not treatment_keywords or pd.isna(treatment_keywords):
-            return []
-        
         chunks = []
-        em_keywords = emergency_keywords.split("|") if emergency_keywords else []
-        tr_keywords = treatment_keywords.split("|") if treatment_keywords else []
+        chunk_size = chunk_size or self.chunk_size
         
-        # Process treatment keywords as primary (since this is treatment-focused data)
-        for i, tr_keyword in enumerate(tr_keywords):
-            tr_pos = text.lower().find(tr_keyword.lower())
+        # Case 1: No keywords present
+        if not emergency_keywords and not treatment_keywords:
+            return []
+            
+        # Case 2: Only emergency keywords (early return)
+        if emergency_keywords and not treatment_keywords:
+            em_chunks = self.create_keyword_centered_chunks(
+                text=text,
+                matched_keywords=emergency_keywords,
+                chunk_size=chunk_size,
+                doc_id=doc_id
+            )
+            for chunk in em_chunks:
+                chunk['source_type'] = 'emergency'
+            return em_chunks
+        
+        # Case 3: Only treatment keywords (early return)
+        if treatment_keywords and not emergency_keywords:
+            tr_chunks = self.create_keyword_centered_chunks(
+                text=text,
+                matched_keywords=treatment_keywords,
+                chunk_size=chunk_size,
+                doc_id=doc_id
+            )
+            for chunk in tr_chunks:
+                chunk['source_type'] = 'treatment'
+                chunk['contains_treatment_kws'] = treatment_keywords.split('|')
+                chunk['contains_emergency_kws'] = []
+                chunk['match_type'] = 'treatment_only'
+            return tr_chunks
+        
+        # Case 4: Both keywords present - separate processing
+        # Process emergency keywords
+        if emergency_keywords:
+            em_chunks = self.create_keyword_centered_chunks(
+                text=text,
+                matched_keywords=emergency_keywords,
+                chunk_size=chunk_size,
+                doc_id=doc_id
+            )
+            for chunk in em_chunks:
+                chunk['source_type'] = 'emergency'
+            chunks.extend(em_chunks)
+        
+        # Process treatment keywords
+        if treatment_keywords:
+            tr_chunks = self.create_keyword_centered_chunks(
+                text=text,
+                matched_keywords=treatment_keywords,
+                chunk_size=chunk_size,
+                doc_id=doc_id
+            )
             
-            if tr_pos != -1:
-                # Find closest emergency keyword for context
-                closest_em_keyword = None
-                closest_distance = float('inf')
+            # Parse keywords for metadata
+            em_kws = emergency_keywords.split('|') if emergency_keywords else []
+            tr_kws = treatment_keywords.split('|') if treatment_keywords else []
+            
+            # Add metadata for each treatment chunk
+            for i, chunk in enumerate(tr_chunks):
+                chunk_text = chunk['text'].lower()
                 
-                for em_keyword in em_keywords:
-                    em_pos = text.lower().find(em_keyword.lower())
-                    if em_pos != -1:
-                        distance = abs(tr_pos - em_pos)
-                        if distance < closest_distance and distance < chunk_size:
-                            closest_distance = distance
-                            closest_em_keyword = em_keyword
+                # Check for keyword presence in chunk text
+                contains_emergency_kws = [
+                    kw for kw in em_kws if kw.lower() in chunk_text
+                ]
+                contains_treatment_kws = [
+                    kw for kw in tr_kws if kw.lower() in chunk_text
+                ]
                 
-                # Calculate chunk boundaries
-                if closest_em_keyword:
-                    # Center between both keywords
-                    em_pos = text.lower().find(closest_em_keyword.lower())
-                    center = (tr_pos + em_pos) // 2
-                else:
-                    # Center on treatment keyword
-                    center = tr_pos
+                # Determine match type based on keyword presence
+                has_emergency = len(contains_emergency_kws) > 0
+                has_treatment = len(contains_treatment_kws) > 0
                 
-                start = max(0, center - chunk_size // 2)
-                end = min(len(text), center + chunk_size // 2)
-                
-                chunk_text = text[start:end].strip()
+                if has_emergency and has_treatment:
+                    match_type = "both"
+                elif has_emergency:
+                    match_type = "emergency_only"
+                elif has_treatment:
+                    match_type = "treatment_only"
+                else:
+                    match_type = "none"
                 
-                if chunk_text:
-                    chunk_info = {
-                        "text": chunk_text,
-                        "primary_keyword": tr_keyword,
-                        "emergency_keywords": emergency_keywords,
-                        "treatment_keywords": treatment_keywords,
-                        "closest_emergency_keyword": closest_em_keyword,
-                        "keyword_distance": closest_distance if closest_em_keyword else None,
-                        "chunk_start": start,
-                        "chunk_end": end,
-                        "chunk_id": f"{doc_id}_treatment_chunk_{i}" if doc_id else f"treatment_chunk_{i}",
-                        "source_doc_id": doc_id
-                    }
-                    chunks.append(chunk_info)
+                # Update chunk metadata
+                chunk.update({
+                    'source_type': 'treatment',
+                    'contains_emergency_kws': contains_emergency_kws,
+                    'contains_treatment_kws': contains_treatment_kws,
+                    'match_type': match_type,
+                    'emergency_keywords': emergency_keywords,  # Store original metadata
+                    'treatment_keywords': treatment_keywords,
+                    'chunk_id': f"{doc_id}_treatment_chunk_{i}" if doc_id else f"treatment_chunk_{i}"
+                })
+            
+            chunks.extend(tr_chunks)
         
         return chunks
     
@@ -308,12 +351,14 @@ class DataProcessor:
         
         all_chunks = []
         
-        # Add progress bar with leave=False to avoid cluttering
+        # Add simplified progress bar
         for idx, row in tqdm(self.emergency_data.iterrows(), 
                         total=len(self.emergency_data),
-                        desc="Processing emergency documents",
-                        unit="doc",
-                        leave=False):
+                        desc="Emergency Processing",
+                        unit="docs",
+                        leave=True,
+                        ncols=80,
+                        mininterval=1.0):
             if pd.notna(row.get('clean_text')) and pd.notna(row.get('matched')):
                 chunks = self.create_keyword_centered_chunks(
                     text=row['clean_text'],
@@ -345,12 +390,14 @@ class DataProcessor:
         
         all_chunks = []
         
-        # Add progress bar with leave=False to avoid cluttering
+        # Add simplified progress bar
         for idx, row in tqdm(self.treatment_data.iterrows(),
                         total=len(self.treatment_data),
-                        desc="Processing treatment documents",
-                        unit="doc",
-                        leave=False):
+                        desc="Treatment Processing",
+                        unit="docs",
+                        leave=True,
+                        ncols=80,
+                        mininterval=1.0):
             if (pd.notna(row.get('clean_text')) and 
                 pd.notna(row.get('treatment_matched'))):
                 
@@ -454,10 +501,12 @@ class DataProcessor:
             logger.info(f"Processing {len(texts)} new {chunk_type} texts in {total_batches} batches...")
             
             for i in tqdm(range(0, len(texts), batch_size), 
-                         desc=f"Embedding {chunk_type} subset",
+                         desc=f"Embedding {chunk_type}",
                          total=total_batches,
-                         unit="batch", 
-                         leave=False):
+                         unit="batches", 
+                         leave=True,
+                         ncols=80,
+                         mininterval=0.5):
                 batch_texts = texts[i:i + batch_size]
                 batch_emb = model.encode(
                     batch_texts,

src/generation.py

@@ -0,0 +1,519 @@
+"""
+OnCall.ai Medical Advice Generation Module
+
+This module handles:  
+1. RAG prompt construction from retrieval results
+2. Medical advice generation using Med42-70B
+3. Response formatting and confidence assessment
+4. Integration with multi-dataset architecture
+
+Author: OnCall.ai Team
+Date: 2025-07-31
+"""
+
+import logging
+from typing import Dict, List, Optional, Any, Union
+from datetime import datetime
+import json
+
+# Import existing LLM client
+from llm_clients import llm_Med42_70BClient
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+class MedicalAdviceGenerator:
+    """
+    Core generation module for medical advice using RAG approach
+    """
+    
+    def __init__(self, llm_client: Optional[llm_Med42_70BClient] = None):
+        """
+        Initialize medical advice generator
+        
+        Args:
+            llm_client: Optional Med42-70B client, creates new if None
+        """
+        self.llm_client = llm_client or llm_Med42_70BClient()
+        
+        # Dataset source priorities for different intentions
+        self.dataset_priorities = {
+            "treatment": {
+                "emergency_subset": 2,
+                "treatment_subset": 4,
+                "symptom_subset": 0,      # Reserved for Dataset B
+                "diagnosis_subset": 0     # Reserved for Dataset B
+            },
+            "diagnosis": {
+                "emergency_subset": 4,
+                "treatment_subset": 2,
+                "symptom_subset": 0,      # Reserved for Dataset B  
+                "diagnosis_subset": 0     # Reserved for Dataset B
+            },
+            # "STAT": {
+            #     # NOTE: Use when query contains urgent indicators like "NOW", "STAT", "critical"
+            #     "emergency_subset": 5,
+            #     "treatment_subset": 1,
+            #     "symptom_subset": 0,      # Reserved for Dataset B
+            #     "diagnosis_subset": 0     # Reserved for Dataset B
+            # }
+        }
+        
+        logger.info("MedicalAdviceGenerator initialized")
+
+    def generate_medical_advice(self, user_query: str, retrieval_results: Dict[str, Any], 
+                               intention: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Complete pipeline: construct prompt → generate advice → format response
+        
+        Args:
+            user_query: Original user medical query
+            retrieval_results: Results from BasicRetrievalSystem.search()
+            intention: Optional query intention ('treatment', 'diagnosis', 'STAT'(tentative))
+            
+        Returns:
+            Dict containing formatted medical advice and metadata
+        """
+        try:
+            logger.info(f"Generating medical advice for query: '{user_query[:50]}...'")
+            start_time = datetime.now()
+            
+            # Step 1: Extract and classify chunks from retrieval results
+            classified_chunks = self._classify_retrieval_chunks(retrieval_results)
+            
+            # Step 2: Build RAG prompt based on intention and chunk classification
+            rag_prompt = self.generate_prompt(user_query, classified_chunks, intention)
+            
+            # Step 3: Generate medical advice using Med42-70B
+            generation_result = self._generate_with_med42(rag_prompt)
+            
+            # Step 4: Format structured response
+            formatted_response = self._format_medical_response(
+                user_query=user_query,
+                generated_advice=generation_result,
+                chunks_used=classified_chunks,
+                intention=intention,
+                processing_time=(datetime.now() - start_time).total_seconds()
+            )
+            
+            processing_duration = formatted_response.get('query_metadata', {}).get('processing_time_seconds', 0)
+            logger.info(f"Medical advice generated successfully in {processing_duration:.3f}s")
+            return formatted_response
+            
+        except Exception as e:
+            logger.error(f"Medical advice generation failed: {e}")
+            return self._generate_error_response(user_query, str(e))
+
+    def generate_prompt(self, user_query: str, classified_chunks: Dict[str, List], 
+                       intention: Optional[str] = None) -> str:
+        """
+        Enhanced prompt generator with flexible dataset integration
+        
+        Args:
+            user_query: User's medical query
+            classified_chunks: Chunks classified by dataset source
+            intention: Query intention if detected
+            
+        Returns:
+            Structured RAG prompt for Med42-70B
+        """
+        logger.info(f"Generating prompt with intention: {intention}")
+        
+        # Extract chunks by dataset source
+        emergency_chunks = classified_chunks.get("emergency_subset", [])
+        treatment_chunks = classified_chunks.get("treatment_subset", [])
+        symptom_chunks = classified_chunks.get("symptom_subset", [])      # Dataset B (future)
+        diagnosis_chunks = classified_chunks.get("diagnosis_subset", [])  # Dataset B (future)
+        
+        # Select chunks based on intention or intelligent defaults
+        selected_chunks = self._select_chunks_by_intention(
+            intention=intention,
+            emergency_chunks=emergency_chunks,
+            treatment_chunks=treatment_chunks,
+            symptom_chunks=symptom_chunks,
+            diagnosis_chunks=diagnosis_chunks
+        )
+        
+        # Build context block from selected chunks
+        context_block = self._build_context_block(selected_chunks)
+        
+        # Construct medical RAG prompt
+        prompt = self._construct_medical_prompt(user_query, context_block, intention)
+        
+        logger.info(f"Generated prompt with {len(selected_chunks)} chunks, {len(context_block)} chars")
+        return prompt
+
+    def _classify_retrieval_chunks(self, retrieval_results: Dict[str, Any]) -> Dict[str, List]:
+        """
+        Classify retrieval chunks by dataset source
+        
+        Args:
+            retrieval_results: Results from BasicRetrievalSystem.search()
+            
+        Returns:
+            Dict mapping dataset sources to chunk lists
+        """
+        classified = {
+            "emergency_subset": [],
+            "treatment_subset": [],
+            "symptom_subset": [],      # Reserved for Dataset B
+            "diagnosis_subset": []     # Reserved for Dataset B
+        }
+        
+        # Process results from current dual-index system
+        processed_results = retrieval_results.get('processed_results', [])
+        
+        for chunk in processed_results:
+            chunk_type = chunk.get('type', 'unknown')
+            
+            # Map current system types to dataset sources
+            if chunk_type == 'emergency':
+                classified["emergency_subset"].append(chunk)
+            elif chunk_type == 'treatment':
+                classified["treatment_subset"].append(chunk)
+            else:
+                # Unknown type, classify by content analysis or default to STAT (tentative)
+                logger.warning(f"Unknown chunk type: {chunk_type}, defaulting to STAT (tentative)")
+                classified["emergency_subset"].append(chunk)
+        
+        # TODO: Future integration point for Dataset B
+        # When Dataset B team provides symptom/diagnosis data:
+        # classified["symptom_subset"] = process_dataset_b_symptoms(retrieval_results)
+        # classified["diagnosis_subset"] = process_dataset_b_diagnosis(retrieval_results)
+        
+        logger.info(f"Classified chunks: Emergency={len(classified['emergency_subset'])}, "
+                   f"Treatment={len(classified['treatment_subset'])}")
+        
+        return classified
+
+    def _select_chunks_by_intention(self, intention: Optional[str], 
+                                   emergency_chunks: List, treatment_chunks: List,
+                                   symptom_chunks: List, diagnosis_chunks: List) -> List:
+        """
+        Select optimal chunk combination based on query intention
+        
+        Args:
+            intention: Detected or specified intention  
+            *_chunks: Chunks from different dataset sources
+            
+        Returns:
+            List of selected chunks for prompt construction
+        """
+        if intention and intention in self.dataset_priorities:
+            # Use predefined priorities for known intentions
+            priorities = self.dataset_priorities[intention]
+            selected_chunks = []
+            
+            # Add chunks according to priority allocation
+            selected_chunks.extend(emergency_chunks[:priorities["emergency_subset"]])
+            selected_chunks.extend(treatment_chunks[:priorities["treatment_subset"]])
+            
+            # TODO: Future Dataset B integration
+            # selected_chunks.extend(symptom_chunks[:priorities["symptom_subset"]])
+            # selected_chunks.extend(diagnosis_chunks[:priorities["diagnosis_subset"]])
+            
+            logger.info(f"Selected chunks by intention '{intention}': {len(selected_chunks)} total")
+            
+        else:
+            # No specific intention - let LLM judge from best available chunks
+            all_chunks = emergency_chunks + treatment_chunks + symptom_chunks + diagnosis_chunks
+            
+            # Sort by relevance (distance) and take top 6
+            all_chunks_sorted = sorted(all_chunks, key=lambda x: x.get("distance", 999))
+            selected_chunks = all_chunks_sorted[:6]
+            
+            logger.info(f"Selected chunks by relevance (no intention): {len(selected_chunks)} total")
+        
+        return selected_chunks
+
+    def _build_context_block(self, selected_chunks: List) -> str:
+        """
+        Build formatted context block from selected chunks
+        
+        Args:
+            selected_chunks: List of selected chunks
+            
+        Returns:
+            Formatted context string for prompt
+        """
+        if not selected_chunks:
+            return "No relevant medical guidelines found."
+        
+        context_parts = []
+        
+        for i, chunk in enumerate(selected_chunks, 1):
+            chunk_text = chunk.get("text", "").strip()
+            chunk_type = chunk.get("type", "unknown")
+            distance = chunk.get("distance", 0)
+            
+            # Format each chunk with metadata
+            context_part = f"""
+[Guideline {i}] (Source: {chunk_type.title()}, Relevance: {1-distance:.3f})
+{chunk_text}
+            """.strip()
+            
+            context_parts.append(context_part)
+        
+        return "\n\n".join(context_parts)
+
+    def _construct_medical_prompt(self, user_query: str, context_block: str, 
+                                 intention: Optional[str]) -> str:
+        """
+        Construct final medical RAG prompt with appropriate framing
+        
+        Args:
+            user_query: Original user query
+            context_block: Formatted context from selected chunks
+            intention: Query intention if detected
+            
+        Returns:
+            Complete RAG prompt for Med42-70B
+        """
+        # Customize prompt based on intention
+        if intention == "treatment":
+            focus_guidance = "Focus on providing specific treatment protocols, management steps, and therapeutic interventions."
+        elif intention == "diagnosis":  
+            focus_guidance = "Focus on differential diagnosis, diagnostic criteria, and assessment approaches."
+        elif intention == "STAT(tentative)":
+            focus_guidance = "Focus on immediate emergency interventions and critical decision-making steps."
+        else:
+            focus_guidance = "Provide comprehensive medical guidance covering both diagnostic and treatment aspects as appropriate."
+        
+        prompt = f"""You are an experienced attending physician providing guidance to a junior clinician in an emergency setting. A colleague is asking for your expert medical opinion.
+
+Clinical Question:
+{user_query}
+
+Relevant Medical Guidelines:
+{context_block}
+
+Instructions:
+{focus_guidance}
+
+Please provide a clear, actionable response that:
+1. Addresses the specific clinical question asked
+2. References relevant evidence from the provided guidelines
+3. Offers practical, step-by-step guidance when appropriate
+4. Maintains appropriate medical caution and emphasizes the need for clinical judgment
+
+Your response should be concise but comprehensive, suitable for immediate clinical application."""
+
+        return prompt
+
+    def _generate_with_med42(self, prompt: str) -> Dict[str, Any]:
+        """
+        Generate medical advice using Med42-70B
+        
+        Args:
+            prompt: Complete RAG prompt
+            
+        Returns:
+            Generation result with metadata
+        """
+        try:
+            logger.info("Calling Med42-70B for medical advice generation")
+            
+            result = self.llm_client.analyze_medical_query(
+                query=prompt,
+                max_tokens=500,  # Adjust based on needs
+                timeout=30.0     # Allow more time for complex medical advice
+            )
+            
+            if result.get('error'):
+                raise Exception(f"Med42-70B generation error: {result['error']}")
+            
+            return result
+            
+        except Exception as e:
+            logger.error(f"Med42-70B generation failed: {e}")
+            raise
+
+    def _format_medical_response(self, user_query: str, generated_advice: Dict[str, Any],
+                                chunks_used: Dict[str, List], intention: Optional[str],
+                                processing_time: float) -> Dict[str, Any]:
+        """
+        Format final medical response with metadata and confidence assessment
+        
+        Args:
+            user_query: Original query
+            generated_advice: Result from Med42-70B
+            chunks_used: Classification of chunks used
+            intention: Detected intention
+            processing_time: Total processing time
+            
+        Returns:
+            Structured medical advice response
+        """
+        # Extract generated content
+        advice_content = generated_advice.get('extracted_condition', '')
+        if not advice_content:
+            advice_content = generated_advice.get('raw_response', 'Unable to generate medical advice.')
+        
+        # Calculate confidence based on available factors
+        confidence_score = self._calculate_confidence_score(generated_advice, chunks_used)
+        
+        # Count chunks used by source
+        chunk_counts = {source: len(chunks) for source, chunks in chunks_used.items()}
+        total_chunks = sum(chunk_counts.values())
+        
+        formatted_response = {
+            "medical_advice": advice_content,
+            "confidence_score": confidence_score,
+            "query_metadata": {
+                "original_query": user_query,
+                "detected_intention": intention,
+                "processing_time_seconds": processing_time,
+                "total_chunks_used": total_chunks,
+                "chunks_by_source": chunk_counts
+            },
+            "generation_metadata": {
+                "model_used": "m42-health/Llama3-Med42-70B",
+                "generation_time": generated_advice.get('latency', 0),
+                "model_confidence": generated_advice.get('confidence', 'unknown'),
+                "timestamp": datetime.now().isoformat()
+            },
+            "sources": {
+                "emergency_sources": len(chunks_used.get("emergency_subset", [])),
+                "treatment_sources": len(chunks_used.get("treatment_subset", [])),
+                "total_sources": total_chunks
+            },
+            "disclaimer": "This advice is for informational purposes only and should not replace professional medical consultation. Always consult with qualified healthcare providers for medical decisions."
+        }
+        
+        return formatted_response
+
+    def _calculate_confidence_score(self, generated_advice: Dict[str, Any], 
+                                   chunks_used: Dict[str, List]) -> float:
+        """
+        Calculate confidence score based on generation quality and source reliability
+        
+        Args:
+            generated_advice: Result from Med42-70B
+            chunks_used: Chunks used in generation
+            
+        Returns:
+            Confidence score between 0.0 and 1.0
+        """
+        confidence_factors = []
+        
+        # Factor 1: Model confidence if available
+        model_confidence = generated_advice.get('confidence', '0.5')
+        try:
+            model_conf_value = float(model_confidence)
+            confidence_factors.append(model_conf_value)
+        except (ValueError, TypeError):
+            confidence_factors.append(0.5)  # Default neutral confidence
+        
+        # Factor 2: Number of sources used (more sources = higher confidence)
+        total_chunks = sum(len(chunks) for chunks in chunks_used.values())
+        source_confidence = min(total_chunks / 6.0, 1.0)  # Normalize to max 6 chunks
+        confidence_factors.append(source_confidence)
+        
+        # Factor 3: Response length (reasonable length indicates comprehensive advice)
+        response_length = len(generated_advice.get('raw_response', ''))
+        length_confidence = min(response_length / 500.0, 1.0)  # Normalize to ~500 chars
+        confidence_factors.append(length_confidence)
+        
+        # Factor 4: Processing success (no errors = higher confidence)
+        if generated_advice.get('error'):
+            confidence_factors.append(0.3)  # Lower confidence if errors occurred
+        else:
+            confidence_factors.append(0.8)  # Higher confidence for clean generation
+        
+        # Calculate weighted average
+        final_confidence = sum(confidence_factors) / len(confidence_factors)
+        
+        # Ensure confidence is within valid range
+        return max(0.1, min(0.95, final_confidence))
+
+    def _generate_error_response(self, user_query: str, error_message: str) -> Dict[str, Any]:
+        """
+        Generate error response when generation fails
+        
+        Args:
+            user_query: Original query
+            error_message: Error details
+            
+        Returns:
+            Error response in standard format
+        """
+        return {
+            "medical_advice": "I apologize, but I encountered an error while processing your medical query. Please try rephrasing your question or contact technical support if the issue persists.",
+            "confidence_score": 0.0,
+            "query_metadata": {
+                "original_query": user_query,
+                "detected_intention": None,
+                "processing_time_seconds": 0.0,
+                "total_chunks_used": 0,
+                "chunks_by_source": {}
+            },
+            "generation_metadata": {
+                "model_used": "m42-health/Llama3-Med42-70B",
+                "error": error_message,
+                "timestamp": datetime.now().isoformat()
+            },
+            "sources": {
+                "emergency_sources": 0,
+                "treatment_sources": 0,
+                "total_sources": 0
+            },
+            "disclaimer": "This system experienced a technical error. Please consult with qualified healthcare providers for medical decisions."
+        }
+
+# Example usage and testing
+def main():
+    """
+    Test the medical advice generation system
+    """
+    # Initialize generator 
+    generator = MedicalAdviceGenerator()
+    
+    # Example retrieval results (simulated)
+    example_retrieval_results = {
+        "processed_results": [
+            {
+                "type": "emergency", 
+                "distance": 0.3,
+                "text": "Acute myocardial infarction requires immediate assessment including ECG, cardiac enzymes, and chest X-ray. Time-sensitive condition requiring rapid intervention.",
+                "matched": "MI|chest pain"
+            },
+            {
+                "type": "treatment",
+                "distance": 0.25, 
+                "text": "Treatment protocol for STEMI includes aspirin 325mg, clopidogrel loading dose, and urgent PCI within 90 minutes when available.",
+                "matched_treatment": "aspirin|PCI|thrombolytic"
+            }
+        ]
+    }
+    
+    # Test queries
+    test_queries = [
+        ("How should I treat a patient with chest pain?", "treatment"),
+        ("What are the signs of acute MI?", "diagnosis"),
+        # ("Emergency management of cardiac arrest", "STAT(tentative)")
+    ]
+    
+    for query, intention in test_queries:
+        print(f"\n{'='*60}")
+        print(f"Testing: {query}")
+        print(f"Intention: {intention}")
+        
+        try:
+            result = generator.generate_medical_advice(
+                user_query=query,
+                retrieval_results=example_retrieval_results,
+                intention=intention
+            )
+            
+            print(f"✅ Success: {result['confidence_score']:.2f} confidence")
+            print(f"Advice: {result['medical_advice'][:200]}...")
+            
+        except Exception as e:
+            print(f"❌ Error: {e}")
+
+if __name__ == "__main__":
+    main()

src/llm_clients.py

@@ -0,0 +1,308 @@
+"""
+OnCall.ai LLM Clients Module
+
+Provides specialized LLM clients for medical query processing.
+
+Author: OnCall.ai Team
+Date: 2025-07-29
+"""
+
+import logging
+import os
+from typing import Dict, Optional, Union
+from huggingface_hub import InferenceClient
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+class llm_Med42_70BClient:
+    def __init__(
+        self, 
+        model_name: str = "m42-health/Llama3-Med42-70B",
+        timeout: float = 30.0
+    ):
+        """
+        Initialize Medical LLM client for query processing.
+        
+        Args:
+            model_name: Hugging Face model name
+            timeout: API call timeout duration
+        
+        Warning: This model should not be used for professional medical advice.
+        """
+        self.logger = logging.getLogger(__name__)
+        self.timeout = timeout
+        
+        # Configure logging to show detailed information
+        logging.basicConfig(
+            level=logging.INFO, 
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        
+        # Get Hugging Face token from environment
+        hf_token = os.getenv('HF_TOKEN')
+        if not hf_token:
+            self.logger.error("HF_TOKEN is missing from environment variables.")
+            raise ValueError(
+                "HF_TOKEN not found in environment variables. "
+                "Please set HF_TOKEN in your .env file or environment. "
+                "Ensure the token is not empty and is correctly set."
+            )
+        
+        try:
+            # Initialize InferenceClient with the new model
+            self.client = InferenceClient(
+                provider="featherless-ai", 
+                api_key=hf_token
+            )
+            
+            self.logger.info(f"Medical LLM client initialized with model: {model_name}")
+            self.logger.warning(
+                "Medical LLM Model: Research tool only. "
+                "Not for professional medical diagnosis."
+            )
+        except Exception as e:
+            self.logger.error(f"Failed to initialize InferenceClient: {str(e)}")
+            self.logger.error(f"Error Type: {type(e).__name__}")
+            self.logger.error(f"Detailed Error: {repr(e)}")
+            raise ValueError(f"Failed to initialize Medical LLM client: {str(e)}") from e
+
+    def analyze_medical_query(
+        self, 
+        query: str, 
+        max_tokens: int = 100, 
+        timeout: Optional[float] = None
+    ) -> Dict[str, Union[str, float]]:
+        """
+        Analyze medical query and extract condition.
+        
+        Args:
+            query: Medical query text
+            max_tokens: Maximum tokens to generate
+            timeout: Specific API call timeout
+        
+        Returns:
+            Extracted medical condition information with latency
+        """
+        import time
+        
+        # Start timing
+        start_time = time.time()
+        
+        try:
+            self.logger.info(f"Calling Medical LLM with query: {query}")
+            
+            # Prepare chat completion request with updated system prompt
+            response = self.client.chat.completions.create(
+                model="m42-health/Llama3-Med42-70B",
+                messages=[
+                    {
+                        "role": "system", 
+                        "content": """You are a medical assistant trained to extract medical conditions.
+
+For medical queries: Extract the most representative medical condition name.
+For non-medical queries: Respond with "NON_MEDICAL_QUERY" and briefly explain why it's not medical.
+
+Examples:
+- Medical: "chest pain" → "Acute Coronary Syndrome"
+- Non-medical: "cooking pasta" → "NON_MEDICAL_QUERY. This is about culinary techniques, not medical conditions."
+
+DO NOT provide medical advice."""
+                    },
+                    {
+                        "role": "user", 
+                        "content": query
+                    }
+                ],
+                max_tokens=max_tokens
+            )
+            
+            # Calculate latency
+            end_time = time.time()
+            latency = end_time - start_time
+            
+            # Extract the response text
+            response_text = response.choices[0].message.content or ""
+            
+            # Log raw response and latency
+            self.logger.info(f"Raw LLM Response: {response_text}")
+            self.logger.info(f"Query Latency: {latency:.4f} seconds")
+            
+            # Extract condition from response
+            extracted_condition = self._extract_condition(response_text)
+            
+            # Log the extracted condition
+            self.logger.info(f"Extracted Condition: {extracted_condition}")
+            
+            return {
+                'extracted_condition': extracted_condition,
+                'confidence': '0.8',
+                'raw_response': response_text,
+                'latency': latency  # Add latency to the return dictionary
+            }
+        
+        except Exception as e:
+            # Calculate latency even for failed requests
+            end_time = time.time()
+            latency = end_time - start_time
+            
+            self.logger.error(f"Medical LLM query error: {str(e)}")
+            self.logger.error(f"Error Type: {type(e).__name__}")
+            self.logger.error(f"Detailed Error: {repr(e)}")
+            self.logger.error(f"Query Latency (on error): {latency:.4f} seconds")
+            
+            # Additional context logging
+            self.logger.error(f"Query that caused error: {query}")
+            
+            return {
+                'extracted_condition': '',
+                'confidence': '0',
+                'error': str(e),
+                'latency': latency  # Include latency even for error cases
+            }
+
+    def _extract_condition(self, response: str) -> str:
+        """
+        Extract medical condition from model response.
+        
+        Args:
+            response: Full model-generated text
+        
+        Returns:
+            Extracted medical condition or empty string if non-medical
+        """
+        # Check if this is a rejection response first
+        if self._is_rejection_response(response):
+            return ""
+        
+        from medical_conditions import CONDITION_KEYWORD_MAPPING
+        
+        # Search in known medical conditions
+        for condition in CONDITION_KEYWORD_MAPPING.keys():
+            if condition.lower() in response.lower():
+                return condition
+        
+        return response.split('\n')[0].strip() or ""
+    
+    def _is_rejection_response(self, response: str) -> bool:
+        """
+        Dual-layer detection: prompt compliance + natural language patterns
+        
+        Args:
+            response: LLM response text
+            
+        Returns:
+            True if response indicates non-medical query rejection
+        """
+        response_upper = response.upper()
+        response_lower = response.lower()
+        
+        # Layer 1: Check for standardized format (if LLM follows prompt)
+        if "NON_MEDICAL_QUERY" in response_upper:
+            return True
+        
+        # Layer 2: Check natural language rejection patterns (fallback)
+        rejection_patterns = [
+            "i do not address",
+            "do not address", 
+            "outside my biomedical scope",
+            "outside my medical scope", 
+            "unrelated to medical conditions",
+            "not about a medical condition",
+            "not a medical condition",
+            "this query is outside",
+            "culinary practice",  # cooking-related
+            "technology trends",  # programming-related
+            "meteorology",        # weather-related
+            "non-medical context"
+        ]
+        
+        return any(pattern in response_lower for pattern in rejection_patterns)
+
+def main():
+    """
+    Test Medical LLM client functionality
+    """
+    import time
+    from datetime import datetime
+
+    # Record total execution start time
+    total_start_time = time.time()
+    execution_start_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    
+    try:
+        print(f"Execution Started at: {execution_start_timestamp}")
+        
+        # Test client initialization
+        client = llm_Med42_70BClient()
+        
+        test_queries = [
+            "patient experiencing chest pain",
+            "sudden weakness on one side",
+            "severe headache with neurological symptoms"
+        ]
+        
+        # Store individual query results
+        query_results = []
+        
+        for query in test_queries:
+            print(f"\nTesting query: {query}")
+            result = client.analyze_medical_query(query)
+            
+            # Store query result
+            query_result = {
+                'query': query,
+                'extracted_condition': result.get('extracted_condition', 'N/A'),
+                'confidence': result.get('confidence', 'N/A'),
+                'latency': result.get('latency', 'N/A')
+            }
+            query_results.append(query_result)
+            
+            # Print individual query results
+            print("Extracted Condition:", query_result['extracted_condition'])
+            print("Confidence:", query_result['confidence'])
+            print(f"Latency: {query_result['latency']:.4f} seconds")
+            
+            if 'error' in result:
+                print("Error:", result['error'])
+            print("---")
+        
+        # Calculate total execution time
+        total_end_time = time.time()
+        total_execution_time = total_end_time - total_start_time
+        execution_end_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        
+        # Print summary
+        print("\n--- Execution Summary ---")
+        print(f"Execution Started at: {execution_start_timestamp}")
+        print(f"Execution Ended at: {execution_end_timestamp}")
+        print(f"Total Execution Time: {total_execution_time:.4f} seconds")
+        
+        # Optional: Return results for potential further processing
+        return {
+            'start_time': execution_start_timestamp,
+            'end_time': execution_end_timestamp,
+            'total_execution_time': total_execution_time,
+            'query_results': query_results
+        }
+            
+    except Exception as e:
+        print(f"Client initialization error: {str(e)}")
+        print("Possible issues:")
+        print("1. Invalid or missing Hugging Face token")
+        print("2. Network connectivity problems")
+        print("3. Model access restrictions")
+        print("\nPlease check your .env file and Hugging Face token.")
+        
+        # Calculate total execution time even in case of error
+        total_end_time = time.time()
+        total_execution_time = total_end_time - total_start_time
+        
+        return {
+            'error': str(e),
+            'total_execution_time': total_execution_time
+        }
+
+if __name__ == "__main__":
+    main()

src/medical_conditions.py

@@ -0,0 +1,99 @@
+"""
+OnCall.ai Medical Conditions Configuration
+
+This module provides centralized configuration for:
+1. Predefined medical conditions
+2. Condition-to-keyword mappings
+3. Fallback condition keywords
+
+Author: OnCall.ai Team
+Date: 2025-07-29
+"""
+
+from typing import Dict, Optional
+
+# Comprehensive Condition-to-Keyword Mapping
+CONDITION_KEYWORD_MAPPING: Dict[str, Dict[str, str]] = {
+    "acute myocardial infarction": {
+        "emergency": "MI|chest pain|cardiac arrest",
+        "treatment": "aspirin|nitroglycerin|thrombolytic|PCI"
+    },
+    "acute stroke": {
+        "emergency": "stroke|neurological deficit|sudden weakness", 
+        "treatment": "tPA|thrombolysis|stroke unit care"
+    },
+    "pulmonary embolism": {
+        "emergency": "chest pain|shortness of breath|sudden dyspnea",
+        "treatment": "anticoagulation|heparin|embolectomy"
+    },
+    # extended from @20250729Test_Retrieval.md 
+    "acute_ischemic_stroke": {
+        "emergency": "ischemic stroke|neurological deficit",
+        "treatment": "tPA|stroke unit management"
+    },
+    "hemorrhagic_stroke": {
+        "emergency": "hemorrhagic stroke|intracranial bleeding",
+        "treatment": "blood pressure control|neurosurgery"
+    },
+    "transient_ischemic_attack": {
+        "emergency": "TIA|temporary stroke symptoms",
+        "treatment": "antiplatelet|lifestyle modification"
+    },
+    "acute_coronary_syndrome": {
+        "emergency": "ACS|chest pain|ECG changes",
+        "treatment": "antiplatelet|statins|cardiac monitoring"
+    }
+}
+
+# Fallback Condition Keywords
+FALLBACK_CONDITION_KEYWORDS: Dict[str, str] = {
+    "acute_ischemic_stroke": "acute ischemic stroke treatment",
+    "hemorrhagic_stroke": "hemorrhagic stroke management", 
+    "transient_ischemic_attack": "TIA treatment protocol",
+    "acute_coronary_syndrome": "ACS treatment guidelines",
+    "stable_angina": "stable angina management",
+    "non_cardiac_chest_pain": "non-cardiac chest pain evaluation",
+    "witnessed_cardiac_arrest": "witnessed cardiac arrest protocol",
+    "unwitnessed_cardiac_arrest": "unwitnessed cardiac arrest management",
+    "post_resuscitation_care": "post-resuscitation care guidelines"
+}
+
+def get_condition_keywords(specific_condition: str) -> Optional[str]:
+    """
+    Retrieve fallback keywords for a specific condition
+    
+    Args:
+        specific_condition: Medical condition name
+    
+    Returns:
+        Corresponding keywords or the original condition
+    """
+    return FALLBACK_CONDITION_KEYWORDS.get(specific_condition, specific_condition)
+
+def validate_condition(condition: str) -> bool:
+    """
+    Check if a condition exists in our predefined mapping
+    
+    Args:
+        condition: Medical condition to validate
+    
+    Returns:
+        Boolean indicating condition validity
+    """
+    return condition.lower() in {k.lower() for k in CONDITION_KEYWORD_MAPPING.keys()}
+
+def get_condition_details(condition: str) -> Optional[Dict[str, str]]:
+    """
+    Retrieve detailed information for a specific condition
+    
+    Args:
+        condition: Medical condition name
+    
+    Returns:
+        Dict with emergency and treatment keywords, or None
+    """
+    normalized_condition = condition.lower()
+    for key, value in CONDITION_KEYWORD_MAPPING.items():
+        if key.lower() == normalized_condition:
+            return value
+    return None 

src/retrieval.py

@@ -0,0 +1,391 @@
+"""
+Basic Retrieval System for OnCall.ai
+
+This module implements the core vector retrieval functionality:
+- Basic vector search
+- Source marking
+- Unified output format
+"""
+
+import numpy as np
+import json
+from pathlib import Path
+from typing import Dict, List, Tuple, Any, Optional
+from sentence_transformers import SentenceTransformer
+from annoy import AnnoyIndex
+import logging
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+class BasicRetrievalSystem:
+    """Basic vector retrieval system for medical documents"""
+    
+    def __init__(self, embedding_dim: int = 768):
+        """
+        Initialize the retrieval system
+        
+        Args:
+            embedding_dim: Dimension of embeddings (default: 768 for PubMedBERT)
+        """
+        self.embedding_dim = embedding_dim
+        self.embedding_model = None
+        self.emergency_index = None
+        self.treatment_index = None
+        self.emergency_chunks = {}
+        self.treatment_chunks = {}
+        
+        # Initialize system
+        self._initialize_system()
+        
+    def _initialize_system(self) -> None:
+        """Initialize embeddings, indices and chunks"""
+        try:
+            logger.info("Initializing retrieval system...")
+            
+            # Initialize embedding model
+            self.embedding_model = SentenceTransformer("NeuML/pubmedbert-base-embeddings")
+            logger.info("Embedding model loaded successfully")
+            
+            # Initialize Annoy indices
+            self.emergency_index = AnnoyIndex(self.embedding_dim, 'angular')
+            self.treatment_index = AnnoyIndex(self.embedding_dim, 'angular')
+            
+            # Load data
+            current_file = Path(__file__)
+            project_root = current_file.parent.parent  # from src to root
+            base_path = project_root / "models"
+            self._load_chunks(base_path)
+            self._load_embeddings(base_path)
+            self._build_or_load_indices(base_path)
+            
+            logger.info("Retrieval system initialized successfully")
+            
+        except Exception as e:
+            logger.error(f"Failed to initialize retrieval system: {e}")
+            raise
+            
+    def _load_chunks(self, base_path: Path) -> None:
+        """Load chunk data from JSON files"""
+        try:
+            # Load emergency chunks
+            with open(base_path / "embeddings" / "emergency_chunks.json", 'r') as f:
+                self.emergency_chunks = json.load(f)
+            
+            # Load treatment chunks
+            with open(base_path / "embeddings" / "treatment_chunks.json", 'r') as f:
+                self.treatment_chunks = json.load(f)
+                
+            logger.info("Chunks loaded successfully")
+            
+        except FileNotFoundError as e:
+            logger.error(f"Chunk file not found: {e}")
+            raise
+        except json.JSONDecodeError as e:
+            logger.error(f"Invalid JSON in chunk file: {e}")
+            raise
+            
+    def _load_embeddings(self, base_path: Path) -> None:
+        """Load pre-computed embeddings"""
+        try:
+            # Load emergency embeddings
+            self.emergency_embeddings = np.load(
+                base_path / "embeddings" / "emergency_embeddings.npy"
+            )
+            
+            # Load treatment embeddings
+            self.treatment_embeddings = np.load(
+                base_path / "embeddings" / "treatment_embeddings.npy"
+            )
+            
+            logger.info("Embeddings loaded successfully")
+            
+        except Exception as e:
+            logger.error(f"Failed to load embeddings: {e}")
+            raise
+            
+    def _build_or_load_indices(self, base_path: Path) -> None:
+        """Build or load Annoy indices"""
+        indices_path = base_path / "indices" / "annoy"
+        emergency_index_path = indices_path / "emergency.ann"
+        treatment_index_path = indices_path / "treatment.ann"
+        
+        try:
+            # Emergency index
+            if emergency_index_path.exists():
+                self.emergency_index.load(str(emergency_index_path))
+                logger.info("Loaded existing emergency index")
+            else:
+                self._build_index(
+                    self.emergency_embeddings,
+                    self.emergency_index,
+                    emergency_index_path
+                )
+                logger.info("Built new emergency index")
+            
+            # Treatment index
+            if treatment_index_path.exists():
+                self.treatment_index.load(str(treatment_index_path))
+                logger.info("Loaded existing treatment index")
+            else:
+                self._build_index(
+                    self.treatment_embeddings,
+                    self.treatment_index,
+                    treatment_index_path
+                )
+                logger.info("Built new treatment index")
+                
+        except Exception as e:
+            logger.error(f"Failed to build/load indices: {e}")
+            raise
+            
+    def _build_index(self, embeddings: np.ndarray, index: AnnoyIndex, 
+                    save_path: Path, n_trees: int = 15) -> None:
+        """
+        Build and save Annoy index
+        
+        Args:
+            embeddings: Embedding vectors
+            index: AnnoyIndex instance
+            save_path: Path to save the index
+            n_trees: Number of trees for Annoy index (default: 15)
+        """
+        try:
+            for i, vec in enumerate(embeddings):
+                index.add_item(i, vec)
+            index.build(n_trees)
+            save_path.parent.mkdir(parents=True, exist_ok=True)
+            index.save(str(save_path))
+            
+        except Exception as e:
+            logger.error(f"Failed to build index: {e}")
+            raise
+            
+    def search(self, query: str, top_k: int = 5) -> Dict[str, Any]:
+        """
+        Perform vector search on both indices
+        
+        Args:
+            query: Search query
+            top_k: Number of results to return from each index
+            
+        Returns:
+            Dict containing search results and metadata
+        """
+        try:
+            # Get query embedding
+            query_embedding = self.embedding_model.encode([query])[0]
+            
+            # Search both indices
+            emergency_results = self._search_index(
+                query_embedding,
+                self.emergency_index,
+                self.emergency_chunks,
+                "emergency",
+                top_k
+            )
+            
+            treatment_results = self._search_index(
+                query_embedding,
+                self.treatment_index,
+                self.treatment_chunks,
+                "treatment",
+                top_k
+            )
+            
+            # Log individual index results
+            logger.info(f"Search results: Emergency={len(emergency_results)}, Treatment={len(treatment_results)}")
+            
+            results = {
+                "query": query,
+                "emergency_results": emergency_results,
+                "treatment_results": treatment_results,
+                "total_results": len(emergency_results) + len(treatment_results)
+            }
+            
+            # Post-process results
+            processed_results = self.post_process_results(results)
+            
+            return processed_results
+            
+        except Exception as e:
+            logger.error(f"Search failed: {e}")
+            raise
+            
+    def _search_index(self, query_embedding: np.ndarray, index: AnnoyIndex,
+                     chunks: Dict, source_type: str, top_k: int) -> List[Dict]:
+        """
+        Search a single index and format results
+        
+        Args:
+            query_embedding: Query vector
+            index: AnnoyIndex to search
+            chunks: Chunk data
+            source_type: Type of source ("emergency" or "treatment")
+            top_k: Number of results to return
+            
+        Returns:
+            List of formatted results
+        """
+        # Get nearest neighbors
+        indices, distances = index.get_nns_by_vector(
+            query_embedding, top_k, include_distances=True
+        )
+        
+        # Format results
+        results = []
+        for idx, distance in zip(indices, distances):
+            chunk_data = chunks[idx]  # chunks is a list, use integer index directly
+            result = {
+                "type": source_type,  # Using 'type' to match metadata
+                "chunk_id": idx,
+                "distance": distance,
+                "text": chunk_data.get("text", ""),
+                "matched": chunk_data.get("matched", ""),
+                "matched_treatment": chunk_data.get("matched_treatment", "")
+            }
+            results.append(result)
+            
+        return results
+        
+    def post_process_results(self, results: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Post-process search results
+        - Remove duplicates
+        - Sort by distance
+        - Add metadata enrichment
+        
+        Args:
+            results: Raw search results
+            
+        Returns:
+            Processed results
+        """
+        try:
+            emergency_results = results["emergency_results"]
+            treatment_results = results["treatment_results"]
+            
+            # Combine all results
+            all_results = emergency_results + treatment_results
+            
+            # Remove duplicates based on exact text matching
+            unique_results = self._remove_duplicates(all_results)
+            
+            # Sort by distance
+            sorted_results = sorted(unique_results, key=lambda x: x["distance"])
+            
+            return {
+                "query": results["query"],
+                "processed_results": sorted_results,
+                "total_results": len(sorted_results),
+                "processing_info": {
+                    "duplicates_removed": len(all_results) - len(unique_results)
+                }
+            }
+            
+        except Exception as e:
+            logger.error(f"Post-processing failed: {e}")
+            raise
+            
+    def _remove_duplicates(self, results: List[Dict]) -> List[Dict]:
+        """
+        Remove duplicate results based on exact text matching
+        
+        Args:
+            results: List of search results
+            
+        Returns:
+            Deduplicated results with logging statistics
+        """
+        original_count = len(results)
+        seen_texts = set()
+        unique_results = []
+        
+        # Sort results by distance (ascending) to keep best matches
+        sorted_results = sorted(results, key=lambda x: x["distance"])
+        
+        logger.info(f"Deduplication: Processing {original_count} results using text matching")
+        
+        for result in sorted_results:
+            text = result["text"]
+            if text not in seen_texts:
+                seen_texts.add(text)
+                unique_results.append(result)
+            else:
+                logger.debug(f"Skipping duplicate text: {text[:50]}...")
+        
+        final_count = len(unique_results)
+        logger.info(f"Deduplication summary: {original_count} → {final_count} results (removed {original_count - final_count})")
+        
+        return unique_results 
+
+    def search_sliding_window_chunks(self, query: str, top_k: int = 5, window_size: int = 256, overlap: int = 64) -> List[Dict[str, Any]]:
+        """
+        Perform semantic search using sliding window chunks
+        
+        Args:
+            query: Search query
+            top_k: Number of top results to return
+            window_size: Size of sliding window chunks
+            overlap: Overlap between sliding windows
+        
+        Returns:
+            List of search results with sliding window chunks
+        """
+        try:
+            # Get query embedding
+            query_embedding = self.embedding_model.encode([query])[0]
+            
+            # Combine emergency and treatment chunks
+            all_chunks = self.emergency_chunks + self.treatment_chunks
+            all_embeddings = np.vstack([self.emergency_embeddings, self.treatment_embeddings])
+            
+            # Compute cosine similarities
+            similarities = [
+                np.dot(query_embedding, chunk_emb) / 
+                (np.linalg.norm(query_embedding) * np.linalg.norm(chunk_emb))
+                for chunk_emb in all_embeddings
+            ]
+            
+            # Sort results by similarity
+            sorted_indices = np.argsort(similarities)[::-1]
+            
+            # Prepare results
+            results = []
+            for idx in sorted_indices[:top_k]:
+                chunk = all_chunks[idx]
+                result = {
+                    'text': chunk.get('text', ''),
+                    'distance': similarities[idx],
+                    'type': 'emergency' if idx < len(self.emergency_chunks) else 'treatment'
+                }
+                results.append(result)
+            
+            logger.info(f"Sliding window search: Found {len(results)} results")
+            return results
+        
+        except Exception as e:
+            logger.error(f"Sliding window search failed: {e}")
+            return [] 
+
+    def search_generic_medical_content(self, query: str, top_k: int = 5) -> List[Dict]:
+        """
+        Perform generic medical content search
+        
+        Args:
+            query: Search query
+            top_k: Number of top results to return
+        
+        Returns:
+            List of search results
+        """
+        try:
+            # re-use search_sliding_window_chunks method
+            return self.search_sliding_window_chunks(query, top_k=top_k)
+        except Exception as e:
+            logger.error(f"Generic medical content search error: {e}")
+            return [] 

src/user_prompt.py

@@ -0,0 +1,562 @@
+"""
+OnCall.ai User Prompt Processing Module
+
+This module handles:
+1. Condition extraction from user queries
+2. Keyword mapping
+3. User confirmation workflow
+4. Fallback mechanisms
+
+Author: OnCall.ai Team
+Date: 2025-07-29
+"""
+
+import logging
+from typing import Dict, Optional, Any, List
+from sentence_transformers import SentenceTransformer
+import numpy as np # Added missing import for numpy
+import os # Added missing import for os
+import json # Added missing import for json
+import re # Added missing import for re
+
+# Import our centralized medical conditions configuration
+from medical_conditions import (
+    CONDITION_KEYWORD_MAPPING, 
+    get_condition_details, 
+    validate_condition
+)
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+class UserPromptProcessor:
+    def __init__(self, llm_client=None, retrieval_system=None):
+        """
+        Initialize UserPromptProcessor with optional LLM and retrieval system
+        
+        Args:
+            llm_client: Optional Llama3-Med42-70B client for advanced condition extraction
+            retrieval_system: Optional retrieval system for semantic search
+        """
+        self.llm_client = llm_client
+        self.retrieval_system = retrieval_system
+        self.embedding_model = SentenceTransformer("NeuML/pubmedbert-base-embeddings")
+        
+        # Add embeddings directory path
+        self.embeddings_dir = os.path.join(os.path.dirname(__file__), '..', 'models', 'embeddings')
+        
+        logger.info("UserPromptProcessor initialized")
+
+    def extract_condition_keywords(self, user_query: str) -> Dict[str, str]:
+        """
+        Extract condition keywords with multi-level fallback
+        
+        Args:
+            user_query: User's medical query
+        
+        Returns:
+            Dict with condition and keywords
+        """
+ 
+        # Level 1: Predefined Mapping (Fast Path)
+        predefined_result = self._predefined_mapping(user_query)
+        if predefined_result:
+            return predefined_result
+        
+        # Level 2: Llama3-Med42-70B Extraction (if available)
+        if self.llm_client:
+            llm_result = self._extract_with_llm(user_query)
+            if llm_result:
+                return llm_result
+        
+        # Level 3: Semantic Search Fallback
+        semantic_result = self._semantic_search_fallback(user_query)
+        if semantic_result:
+            return semantic_result
+        
+        # Level 4: Medical Query Validation
+        # Only validate if previous levels failed - speed optimization
+        validation_result = self.validate_medical_query(user_query)
+        if validation_result:  # If validation fails (returns non-None)
+            return validation_result
+        
+        # Level 5: Generic Medical Search (after validation passes)
+        generic_result = self._generic_medical_search(user_query)
+        if generic_result:
+            return generic_result
+        
+        # No match found
+        
+        return {
+            'condition': '',
+            'emergency_keywords': '',
+            'treatment_keywords': ''
+        }
+
+    def _predefined_mapping(self, user_query: str) -> Optional[Dict[str, str]]:
+        """
+        Fast predefined condition mapping
+        
+        Args:
+            user_query: User's medical query
+        
+        Returns:
+            Mapped condition keywords or None
+        """
+        query_lower = user_query.lower()
+        
+        for condition, mappings in CONDITION_KEYWORD_MAPPING.items():
+            if condition.lower() in query_lower:
+                logger.info(f"Matched predefined condition: {condition}")
+                return {
+                    'condition': condition,
+                    'emergency_keywords': mappings['emergency'],
+                    'treatment_keywords': mappings['treatment']
+                }
+        
+        return None
+
+    def _extract_with_llm(self, user_query: str) -> Optional[Dict[str, str]]:
+        """
+        Use Llama3-Med42-70B for advanced condition extraction
+        
+        Args:
+            user_query: User's medical query
+        
+        Returns:
+            Dict with condition and keywords, or None
+        """
+        if not self.llm_client:
+            return None
+        
+        try:
+            llama_response = self.llm_client.analyze_medical_query(
+                query=user_query,
+                max_tokens=100,
+                timeout=2.0
+            )
+            
+            extracted_condition = llama_response.get('extracted_condition', '')
+            
+            if extracted_condition and validate_condition(extracted_condition):
+                condition_details = get_condition_details(extracted_condition)
+                if condition_details:
+                    return {
+                        'condition': extracted_condition,
+                        'emergency_keywords': condition_details.get('emergency', ''),
+                        'treatment_keywords': condition_details.get('treatment', '')
+                    }
+            
+            return None
+        
+        except Exception as e:
+            logger.error(f"Llama3-Med42-70B condition extraction error: {e}")
+            return None
+
+    def _semantic_search_fallback(self, user_query: str) -> Optional[Dict[str, str]]:
+        """
+        Perform semantic search for condition extraction using sliding window chunks
+        
+        Args:
+            user_query: User's medical query
+        
+        Returns:
+            Dict with condition and keywords, or None
+        """
+        logger.info(f"Starting semantic search fallback for query: '{user_query}'")
+        
+        if not self.retrieval_system:
+            logger.warning("No retrieval system available for semantic search")
+            return None
+        
+        try:
+            # Perform semantic search on sliding window chunks
+            semantic_results = self.retrieval_system.search_sliding_window_chunks(user_query)
+            
+            logger.info(f"Semantic search returned {len(semantic_results)} results")
+            
+            if semantic_results:
+                # Extract condition from top semantic result
+                top_result = semantic_results[0]
+                condition = self._infer_condition_from_text(top_result['text'])
+                
+                logger.info(f"Inferred condition: {condition}")
+                
+                if condition and validate_condition(condition):
+                    condition_details = get_condition_details(condition)
+                    if condition_details:
+                        result = {
+                            'condition': condition,
+                            'emergency_keywords': condition_details.get('emergency', ''),
+                            'treatment_keywords': condition_details.get('treatment', ''),
+                            'semantic_confidence': top_result.get('distance', 0)
+                        }
+                    
+                    logger.info(f"Semantic search successful. Condition: {condition}, "
+                                f"Confidence: {result['semantic_confidence']}")
+                    return result
+                else:
+                    logger.warning(f"Condition validation failed for: {condition}")
+            
+            logger.info("No suitable condition found in semantic search")
+            return None
+        
+        except Exception as e:
+            logger.error(f"Semantic search fallback error: {e}", exc_info=True)
+            return None
+
+    def _generic_medical_search(self, user_query: str) -> Optional[Dict[str, str]]:
+        """
+        Perform generic medical search as final fallback
+        
+        Args:
+            user_query: User's medical query
+        
+        Returns:
+            Dict with generic medical keywords
+        """
+        generic_medical_terms = [
+            "medical", "treatment", "management", "protocol",
+            "guidelines", "emergency", "acute", "chronic"
+        ]
+        
+        generic_query = f"{user_query} medical treatment"
+        
+        try:
+            # Perform generic medical search
+            generic_results = self.retrieval_system.search_generic_medical_content(generic_query)
+            
+            if generic_results:
+                return {
+                    'condition': 'generic medical query',
+                    'emergency_keywords': 'medical|emergency',
+                    'treatment_keywords': 'treatment|management',
+                    'generic_confidence': 0.5
+                }
+            
+            return None 
+        except Exception as e:
+            logger.error(f"Generic medical search error: {e}")
+            return None
+
+    def _infer_condition_from_text(self, text: str) -> Optional[str]:
+        """
+        Infer medical condition from text using embedding similarity
+        
+        Args:
+            text: Input medical text
+        
+        Returns:
+            Inferred condition or None
+        """
+        # Implement a simple condition inference using embedding similarity
+        # This is a placeholder and would need more sophisticated implementation
+        conditions = list(CONDITION_KEYWORD_MAPPING.keys())
+        text_embedding = self.embedding_model.encode(text)
+        condition_embeddings = [self.embedding_model.encode(condition) for condition in conditions]
+        
+        similarities = [
+            np.dot(text_embedding, condition_emb) / 
+            (np.linalg.norm(text_embedding) * np.linalg.norm(condition_emb))
+            for condition_emb in condition_embeddings
+        ]
+        
+        max_similarity_index = np.argmax(similarities)
+        return conditions[max_similarity_index] if similarities[max_similarity_index] > 0.7 else None
+
+    def validate_keywords(self, keywords: Dict[str, str]) -> bool:
+        """
+        Validate if extracted keywords exist in our medical indices
+        
+        Args:
+            keywords: Dict of emergency and treatment keywords
+        
+        Returns:
+            Boolean indicating keyword validity
+        """
+        emergency_kws = keywords.get('emergency_keywords', '').split('|')
+        treatment_kws = keywords.get('treatment_keywords', '').split('|')
+        
+        # Basic validation: check if any keyword is non-empty
+        return any(kw.strip() for kw in emergency_kws + treatment_kws)
+
+    def _check_keyword_in_index(self, keyword: str, index_type: str) -> bool:
+        """
+        Check if a keyword exists in the specified medical index
+        
+        Args:
+            keyword: Keyword to check
+            index_type: Type of index ('emergency' or 'treatment')
+        
+        Returns:
+            Boolean indicating keyword existence in the index
+        """
+        # Validate input parameters
+        if not keyword or not index_type:
+            logger.warning(f"Invalid input: keyword='{keyword}', index_type='{index_type}'")
+            return False
+        
+        # Supported index types
+        valid_index_types = ['emergency', 'treatment']
+        if index_type not in valid_index_types:
+            logger.error(f"Unsupported index type: {index_type}")
+            return False
+        
+        try:
+            # Construct path to chunks file
+            chunks_path = os.path.join(self.embeddings_dir, f"{index_type}_chunks.json")
+            
+            # Check file existence
+            if not os.path.exists(chunks_path):
+                logger.error(f"Index file not found: {chunks_path}")
+                return False
+            
+            # Load chunks with error handling
+            with open(chunks_path, 'r', encoding='utf-8') as f:
+                chunks = json.load(f)
+            
+            # Normalize keyword for flexible matching
+            keyword_lower = keyword.lower().strip()
+            
+            # Advanced keyword matching
+            for chunk in chunks:
+                chunk_text = chunk.get('text', '').lower()
+                
+                # Exact match
+                if keyword_lower in chunk_text:
+                    logger.info(f"Exact match found for '{keyword}' in {index_type} index")
+                    return True
+                
+                # Partial match with word boundaries
+                if re.search(r'\b' + re.escape(keyword_lower) + r'\b', chunk_text):
+                    logger.info(f"Partial match found for '{keyword}' in {index_type} index")
+                    return True
+            
+            # No match found
+            logger.info(f"No match found for '{keyword}' in {index_type} index")
+            return False
+        
+        except json.JSONDecodeError:
+            logger.error(f"Invalid JSON in {chunks_path}")
+            return False
+        except IOError as e:
+            logger.error(f"IO error reading {chunks_path}: {e}")
+            return False
+        except Exception as e:
+            logger.error(f"Unexpected error in _check_keyword_in_index: {e}")
+            return False
+
+    def handle_user_confirmation(self, extracted_info: Dict[str, str]) -> Dict[str, Any]:
+        """
+        Handle user confirmation for extracted condition and keywords
+        
+        Args:
+            extracted_info: Dict with condition and keyword information
+        
+        Returns:
+            Dict with confirmation status and options
+        """
+        # If no condition found, request user to rephrase
+        if not extracted_info.get('condition'):
+            return {
+                'type': 'rephrase_needed',
+                'message': "Could not identify a specific medical condition. Please rephrase your query.",
+                'suggestions': [
+                    "Try: 'how to treat chest pain'",
+                    "Try: 'acute stroke management'",
+                    "Try: 'pulmonary embolism treatment'"
+                ]
+            }
+        
+        # Prepare confirmation message
+        confirmation_message = f"""
+I understand you're asking about: "{extracted_info.get('condition', 'Unknown Condition')}"
+
+Extracted Keywords:
+- Emergency: {extracted_info.get('emergency_keywords', 'None')}
+- Treatment: {extracted_info.get('treatment_keywords', 'None')}
+
+Please confirm:
+1) Yes, proceed with search
+2) No, please rephrase my query
+3) Modify keywords
+        """
+        
+        return {
+            'type': 'confirmation_needed',
+            'message': confirmation_message,
+            'extracted_info': extracted_info
+        }
+
+    def _handle_matching_failure_level1(self, condition: str) -> Optional[Dict[str, Any]]:
+        """
+        Level 1 Fallback: Loose keyword matching for medical conditions
+        
+        Args:
+            condition: The condition to match loosely
+        
+        Returns:
+            Dict with matched keywords or None
+        """
+        # Predefined loose matching keywords for different medical domains
+        loose_medical_keywords = {
+            'emergency': [
+                'urgent', 'critical', 'severe', 'acute', 
+                'immediate', 'life-threatening', 'emergency'
+            ],
+            'treatment': [
+                'manage', 'cure', 'heal', 'recover', 
+                'therapy', 'medication', 'intervention'
+            ]
+        }
+        
+        # Normalize condition
+        condition_lower = condition.lower().strip()
+        
+        # Check emergency keywords
+        emergency_matches = [
+            kw for kw in loose_medical_keywords['emergency'] 
+            if kw in condition_lower
+        ]
+        
+        # Check treatment keywords
+        treatment_matches = [
+            kw for kw in loose_medical_keywords['treatment'] 
+            if kw in condition_lower
+        ]
+        
+        # If matches found, return result
+        if emergency_matches or treatment_matches:
+            logger.info(f"Loose keyword match for condition: {condition}")
+            return {
+                'type': 'loose_keyword_match',
+                'condition': condition,
+                'emergency_keywords': '|'.join(emergency_matches),
+                'treatment_keywords': '|'.join(treatment_matches),
+                'confidence': 0.5  # Lower confidence due to loose matching
+            }
+        
+        # No loose matches found
+        logger.info(f"No loose keyword match for condition: {condition}")
+        return None
+
+    def validate_medical_query(self, user_query: str) -> Dict[str, Any]:
+        """
+        Validate if the query is a medical-related query using Llama3-Med42-70B multi-layer verification
+        
+        Args:
+            user_query: User's input query
+        
+        Returns:
+            Dict with validation result or None if medical query
+        """
+        # Expanded medical keywords covering comprehensive medical terminology
+        predefined_medical_keywords = {
+            # Symptoms and signs
+            'pain', 'symptom', 'ache', 'fever', 'inflammation', 
+            'bleeding', 'swelling', 'rash', 'bruise', 'wound',
+            
+            # Medical professional terms
+            'disease', 'condition', 'syndrome', 'disorder', 
+            'medical', 'health', 'diagnosis', 'treatment', 
+            'therapy', 'medication', 'prescription',
+            
+            # Body systems and organs
+            'heart', 'lung', 'brain', 'kidney', 'liver', 
+            'blood', 'nerve', 'muscle', 'bone', 'joint',
+            
+            # Medical actions
+            'examine', 'check', 'test', 'scan', 'surgery', 
+            'operation', 'emergency', 'urgent', 'critical',
+            
+            # Specific medical fields
+            'cardiology', 'neurology', 'oncology', 'pediatrics', 
+            'psychiatry', 'dermatology', 'orthopedics'
+        }
+        
+        # Check if query contains predefined medical keywords
+        query_lower = user_query.lower()
+        if any(kw in query_lower for kw in predefined_medical_keywords):
+            return None  # Validated by predefined keywords
+        
+        try:
+            # Ensure Llama3-Med42-70B client is properly initialized
+            if not hasattr(self, 'llm_client') or self.llm_client is None:
+                self.logger.warning("Llama3-Med42-70B client not initialized")
+                return self._generate_invalid_query_response()
+            
+            # Use Llama3-Med42-70B for final medical query determination
+            llama_result = self.llm_client.analyze_medical_query(
+                query=user_query,
+                max_tokens=100  # Limit tokens for efficiency
+            )
+            
+            # If Llama3-Med42-70B successfully extracts a medical condition
+            if llama_result.get('extracted_condition'):
+                return None  # Validated by Llama3-Med42-70B
+            
+        except Exception as e:
+            # Log Llama3-Med42-70B analysis failure without blocking the process
+            self.logger.warning(f"Llama3-Med42-70B query validation failed: {e}")
+        
+        # If no medical relevance is found
+        return self._generate_invalid_query_response()
+
+    def _generate_invalid_query_response(self) -> Dict[str, Any]:
+        """
+        Generate response for non-medical queries
+        
+        Returns:
+            Dict with invalid query guidance
+        """
+        return {
+            'type': 'invalid_query',
+            'message': "This is OnCall.AI, a clinical medical assistance platform. "
+                       "Please input a medical problem you need help resolving. "
+                       "\n\nExamples:\n"
+                       "- 'I'm experiencing chest pain'\n"
+                       "- 'What are symptoms of stroke?'\n"
+                       "- 'How to manage acute asthma?'\n"
+                       "- 'I have a persistent headache'"
+        }
+
+def main():
+    """
+    Example usage and testing of UserPromptProcessor with Llama3-Med42-70B
+    Demonstrates condition extraction and query validation
+    """
+    from .retrieval import BasicRetrievalSystem
+
+    # use relative import to avoid circular import
+    from .llm_clients import llm_Med42_70BClient
+    
+    # Initialize LLM client
+    llm_client = llm_Med42_70BClient()
+    retrieval_system = BasicRetrievalSystem()
+    
+    # Initialize UserPromptProcessor with the LLM client
+    processor = UserPromptProcessor(
+        llm_client=llm_client, retrieval_system=retrieval_system
+    )
+    
+    # Update test cases with more representative medical queries
+    test_queries = [
+        "patient with severe chest pain and shortness of breath",
+        "sudden neurological symptoms suggesting stroke",
+        "persistent headache with vision changes"
+    ]
+    
+    for query in test_queries:
+        print(f"\nQuery: {query}")
+        result = processor.extract_condition_keywords(query)
+        print("Extracted Keywords:", result)
+        
+        confirmation = processor.handle_user_confirmation(result)
+        print("Confirmation:", confirmation['message'])
+
+if __name__ == "__main__":
+    main() 

test_retrieval_pipeline.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+"""
+Test script for OnCall.ai retrieval pipeline
+
+This script tests the complete flow:
+user_input → user_prompt.py → retrieval.py
+
+Author: OnCall.ai Team
+Date: 2025-07-30
+"""
+
+import sys
+import os
+from pathlib import Path
+import logging
+import json
+from datetime import datetime
+
+# Add src directory to Python path
+sys.path.append(os.path.join(os.path.dirname(__file__), 'src'))
+
+# Import our modules
+from user_prompt import UserPromptProcessor
+from retrieval import BasicRetrievalSystem
+from llm_clients import llm_Med42_70BClient
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(),
+        logging.FileHandler('test_retrieval_pipeline.log')
+    ]
+)
+logger = logging.getLogger(__name__)
+
+def test_retrieval_pipeline():
+    """
+    Test the complete retrieval pipeline
+    """
+    print("="*60)
+    print("OnCall.ai Retrieval Pipeline Test")
+    print("="*60)
+    print(f"Test started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    print()
+    
+    try:
+        # Initialize components
+        print("🔧 Initializing components...")
+        
+        # Initialize LLM client
+        llm_client = llm_Med42_70BClient()
+        print("✅ LLM client initialized")
+        
+        # Initialize retrieval system
+        retrieval_system = BasicRetrievalSystem()
+        print("✅ Retrieval system initialized")
+        
+        # Initialize user prompt processor
+        user_prompt_processor = UserPromptProcessor(
+            llm_client=llm_client,
+            retrieval_system=retrieval_system
+        )
+        print("✅ User prompt processor initialized")
+        print()
+        
+        # Test queries
+        test_queries = [
+            "how to treat acute MI?",
+            "patient with chest pain and shortness of breath",
+            "sudden neurological symptoms suggesting stroke",
+            "acute stroke management protocol"
+        ]
+        
+        results = []
+        
+        for i, query in enumerate(test_queries, 1):
+            print(f"🔍 Test {i}/{len(test_queries)}: Testing query: '{query}'")
+            print("-" * 50)
+            
+            try:
+                # Step 1: Extract condition keywords
+                print("Step 1: Extracting condition keywords...")
+                condition_result = user_prompt_processor.extract_condition_keywords(query)
+                
+                print(f"  Condition: {condition_result.get('condition', 'None')}")
+                print(f"  Emergency keywords: {condition_result.get('emergency_keywords', 'None')}")
+                print(f"  Treatment keywords: {condition_result.get('treatment_keywords', 'None')}")
+                
+                if not condition_result.get('condition'):
+                    print("  ⚠️  No condition extracted, skipping retrieval")
+                    continue
+                
+                # Step 2: User confirmation (simulated)
+                print("\nStep 2: User confirmation (simulated as 'yes')")
+                confirmation = user_prompt_processor.handle_user_confirmation(condition_result)
+                print(f"  Confirmation type: {confirmation.get('type', 'Unknown')}")
+                
+                # Step 3: Perform retrieval
+                print("\nStep 3: Performing retrieval...")
+                search_query = f"{condition_result.get('emergency_keywords', '')} {condition_result.get('treatment_keywords', '')}".strip()
+                
+                if not search_query:
+                    search_query = condition_result.get('condition', query)
+                
+                print(f"  Search query: '{search_query}'")
+                
+                retrieval_results = retrieval_system.search(search_query, top_k=5)
+                
+                # Display results
+                print(f"\n📊 Retrieval Results:")
+                print(f"  Total results: {retrieval_results.get('total_results', 0)}")
+                
+                emergency_results = retrieval_results.get('emergency_results', [])
+                treatment_results = retrieval_results.get('treatment_results', [])
+                
+                print(f"  Emergency results: {len(emergency_results)}")
+                print(f"  Treatment results: {len(treatment_results)}")
+                
+                # Show top results
+                if 'processed_results' in retrieval_results:
+                    processed_results = retrieval_results['processed_results'][:3]  # Show top 3
+                    print(f"\n  Top {len(processed_results)} results:")
+                    for j, result in enumerate(processed_results, 1):
+                        print(f"    {j}. Type: {result.get('type', 'Unknown')}")
+                        print(f"       Distance: {result.get('distance', 'Unknown'):.4f}")
+                        print(f"       Text preview: {result.get('text', '')[:100]}...")
+                        print(f"       Matched: {result.get('matched', 'None')}")
+                        print(f"       Treatment matched: {result.get('matched_treatment', 'None')}")
+                        print()
+                
+                # Store results for summary
+                test_result = {
+                    'query': query,
+                    'condition_extracted': condition_result.get('condition', ''),
+                    'emergency_keywords': condition_result.get('emergency_keywords', ''),
+                    'treatment_keywords': condition_result.get('treatment_keywords', ''),
+                    'search_query': search_query,
+                    'total_results': retrieval_results.get('total_results', 0),
+                    'emergency_count': len(emergency_results),
+                    'treatment_count': len(treatment_results),
+                    'success': True
+                }
+                results.append(test_result)
+                
+                print("✅ Test completed successfully")
+                
+            except Exception as e:
+                logger.error(f"Error in test {i}: {e}", exc_info=True)
+                test_result = {
+                    'query': query,
+                    'error': str(e),
+                    'success': False
+                }
+                results.append(test_result)
+                print(f"❌ Test failed: {e}")
+            
+            print("\n" + "="*60 + "\n")
+        
+        # Print summary
+        print_test_summary(results)
+        
+        # Save results to file
+        save_test_results(results)
+        
+        return results
+        
+    except Exception as e:
+        logger.error(f"Critical error in pipeline test: {e}", exc_info=True)
+        print(f"❌ Critical error: {e}")
+        return []
+
+def print_test_summary(results):
+    """Print test summary"""
+    print("📋 TEST SUMMARY")
+    print("="*60)
+    
+    successful_tests = [r for r in results if r.get('success', False)]
+    failed_tests = [r for r in results if not r.get('success', False)]
+    
+    print(f"Total tests: {len(results)}")
+    print(f"Successful: {len(successful_tests)}")
+    print(f"Failed: {len(failed_tests)}")
+    print(f"Success rate: {len(successful_tests)/len(results)*100:.1f}%")
+    print()
+    
+    if successful_tests:
+        print("✅ Successful tests:")
+        for result in successful_tests:
+            print(f"  - '{result['query']}'")
+            print(f"    Condition: {result.get('condition_extracted', 'None')}")
+            print(f"    Results: {result.get('total_results', 0)} total "
+                  f"({result.get('emergency_count', 0)} emergency, "
+                  f"{result.get('treatment_count', 0)} treatment)")
+            print()
+    
+    if failed_tests:
+        print("❌ Failed tests:")
+        for result in failed_tests:
+            print(f"  - '{result['query']}': {result.get('error', 'Unknown error')}")
+        print()
+
+def save_test_results(results):
+    """Save test results to JSON file"""
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    filename = f"test_results_{timestamp}.json"
+    
+    try:
+        with open(filename, 'w', encoding='utf-8') as f:
+            json.dump({
+                'timestamp': datetime.now().isoformat(),
+                'test_results': results
+            }, f, indent=2, ensure_ascii=False)
+        
+        print(f"📁 Test results saved to: {filename}")
+        
+    except Exception as e:
+        logger.error(f"Failed to save test results: {e}")
+        print(f"⚠️  Failed to save test results: {e}")
+
+if __name__ == "__main__":
+    test_retrieval_pipeline()

tests/requirements.txt

@@ -0,0 +1,95 @@
+aiofiles==24.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.12.14
+aiosignal==1.4.0
+annotated-types==0.7.0
+annoy==1.17.3
+anyio==4.9.0
+attrs==25.3.0
+Brotli==1.1.0
+certifi==2025.7.14
+charset-normalizer==3.4.2
+click==8.2.1
+contourpy==1.3.2
+cycler==0.12.1
+datasets==4.0.0
+dill==0.3.8
+distro==1.9.0
+fastapi==0.116.1
+ffmpy==0.6.0
+filelock==3.18.0
+fonttools==4.59.0
+frozenlist==1.7.0
+fsspec==2025.3.0
+gradio==5.38.0
+gradio_client==1.11.0
+groovy==0.1.2
+h11==0.16.0
+hf-xet==1.1.5
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.33.4
+idna==3.10
+iniconfig==2.1.0
+Jinja2==3.1.6
+jiter==0.10.0
+joblib==1.5.1
+kiwisolver==1.4.8
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+matplotlib==3.10.3
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.6.3
+multiprocess==0.70.16
+networkx==3.5
+numpy==2.3.1
+openai==1.97.0
+orjson==3.11.0
+packaging==25.0
+pandas==2.3.1
+pillow==11.3.0
+pluggy==1.6.0
+propcache==0.3.2
+pyarrow==20.0.0
+pydantic==2.11.7
+pydantic_core==2.33.2
+pydub==0.25.1
+Pygments==2.19.2
+pyparsing==3.2.3
+pytest==8.4.1
+python-dateutil==2.9.0.post0
+python-multipart==0.0.20
+pytz==2025.2
+PyYAML==6.0.2
+regex==2024.11.6
+requests==2.32.4
+rich==14.0.0
+ruff==0.12.4
+safehttpx==0.1.6
+safetensors==0.5.3
+scikit-learn==1.7.1
+scipy==1.16.1
+seaborn==0.13.2
+semantic-version==2.10.0
+sentence-transformers==3.0.1
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+starlette==0.47.2
+sympy==1.14.0
+threadpoolctl==3.6.0
+tokenizers==0.21.2
+tomlkit==0.13.3
+torch==2.7.1
+tqdm==4.67.1
+transformers==4.53.2
+typer==0.16.0
+typing-inspection==0.4.1
+typing_extensions==4.14.1
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.35.0
+websockets==15.0.1
+xxhash==3.5.0
+yarl==1.20.1

tests/result_of_test_multlevel_fallback_validation.md

@@ -0,0 +1,570 @@
+🏥 OnCall.ai Multilevel Fallback Validation Test
+============================================================
+🔧 Initializing Components for Multilevel Fallback Test...
+------------------------------------------------------------
+1. Initializing Llama3-Med42-70B Client...
+2025-07-31 07:12:17,625 - llm_clients - INFO - Medical LLM client initialized with model: m42-health/Llama3-Med42-70B
+2025-07-31 07:12:17,626 - llm_clients - WARNING - Medical LLM Model: Research tool only. Not for professional medical diagnosis.
+   ✅ LLM client initialized
+2. Initializing Retrieval System...
+2025-07-31 07:12:17,626 - retrieval - INFO - Initializing retrieval system...
+2025-07-31 07:12:17,637 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-07-31 07:12:17,637 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: NeuML/pubmedbert-base-embeddings
+2025-07-31 07:12:20,936 - retrieval - INFO - Embedding model loaded successfully
+2025-07-31 07:12:22,314 - retrieval - INFO - Chunks loaded successfully
+2025-07-31 07:12:22,418 - retrieval - INFO - Embeddings loaded successfully
+2025-07-31 07:12:22,419 - retrieval - INFO - Loaded existing emergency index
+2025-07-31 07:12:22,420 - retrieval - INFO - Loaded existing treatment index
+2025-07-31 07:12:22,420 - retrieval - INFO - Retrieval system initialized successfully
+   ✅ Retrieval system initialized
+3. Initializing User Prompt Processor...
+2025-07-31 07:12:22,420 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-07-31 07:12:22,420 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: NeuML/pubmedbert-base-embeddings
+2025-07-31 07:12:24,622 - user_prompt - INFO - UserPromptProcessor initialized
+   ✅ User prompt processor initialized
+
+🎉 All components initialized successfully!
+
+🚀 Starting Multilevel Fallback Test Suite
+Total test cases: 13
+Test started at: 2025-07-31 07:12:17
+================================================================================
+
+🔍 level1_001: Level 1: Direct predefined condition match
+Query: 'acute myocardial infarction treatment'
+Expected Level: 1
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:12:24,623 - user_prompt - INFO - Matched predefined condition: acute myocardial infarction
+   ✅ Detected Level: 1
+   Condition: acute myocardial infarction
+   Emergency Keywords: MI|chest pain|cardiac arrest
+   Treatment Keywords: aspirin|nitroglycerin|thrombolytic|PCI
+   Execution Time: 0.000s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level1_002: Level 1: Predefined stroke condition
+Query: 'how to manage acute stroke?'
+Expected Level: 1
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:12:24,623 - user_prompt - INFO - Matched predefined condition: acute stroke
+   ✅ Detected Level: 1
+   Condition: acute stroke
+   Emergency Keywords: stroke|neurological deficit|sudden weakness
+   Treatment Keywords: tPA|thrombolysis|stroke unit care
+   Execution Time: 0.000s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level1_003: Level 1: Predefined PE condition
+Query: 'pulmonary embolism emergency protocol'
+Expected Level: 1
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:12:24,623 - user_prompt - INFO - Matched predefined condition: pulmonary embolism
+   ✅ Detected Level: 1
+   Condition: pulmonary embolism
+   Emergency Keywords: chest pain|shortness of breath|sudden dyspnea
+   Treatment Keywords: anticoagulation|heparin|embolectomy
+   Execution Time: 0.000s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level2_001: Level 2: Symptom-based query requiring LLM analysis
+Query: 'patient with severe crushing chest pain radiating to left arm'
+Expected Level: 2
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:12:24,623 - llm_clients - INFO - Calling Medical LLM with query: patient with severe crushing chest pain radiating to left arm
+2025-07-31 07:12:47,629 - llm_clients - INFO - Raw LLM Response: Acute Myocardial Infarction (STEMI) - considering "severe crushing chest pain" and radiation to the left arm, which are classic symptoms of a heart attack specifically involving ST-elevation (STEMI type), indicating complete blockage of a coronary artery. However, please note that as an AI assistant, I don't diagnose; this interpretation is based on common clinical presentation. A healthcare provider should perform an ECG and other tests for confirmation.
+2025-07-31 07:12:47,630 - llm_clients - INFO - Query Latency: 23.0064 seconds
+2025-07-31 07:12:47,630 - llm_clients - INFO - Extracted Condition: acute myocardial infarction
+   ✅ Detected Level: 1
+   Condition: acute myocardial infarction
+   Emergency Keywords: MI|chest pain|cardiac arrest
+   Treatment Keywords: aspirin|nitroglycerin|thrombolytic|PCI
+   Execution Time: 23.008s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level2_002: Level 2: Neurological symptoms requiring LLM
+Query: 'sudden onset weakness on right side with speech difficulty'
+Expected Level: 2
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:12:47,631 - llm_clients - INFO - Calling Medical LLM with query: sudden onset weakness on right side with speech difficulty
+2025-07-31 07:12:56,760 - llm_clients - INFO - Raw LLM Response: Cerebrovascular Accident (CVA), or Acute Ischemic Stroke (specifically, with right hemiparesis and aphasia) 
+
+- This diagnosis represents the most likely condition given the sudden onset of right-sided weakness (hemiparesis) and speech difficulty (aphasia). An ischemic stroke occurs when blood flow to a part of the brain is blocked, typically by a thrombus or embolus, causing damage to brain tissue and resulting in neurological deficits. Immediate medical
+2025-07-31 07:12:56,760 - llm_clients - INFO - Query Latency: 9.1288 seconds
+2025-07-31 07:12:56,760 - llm_clients - INFO - Extracted Condition: Cerebrovascular Accident (CVA), or Acute Ischemic Stroke (specifically, with right hemiparesis and aphasia)
+2025-07-31 07:12:56,760 - user_prompt - INFO - Starting semantic search fallback for query: 'sudden onset weakness on right side with speech difficulty'
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  1.66it/s]
+2025-07-31 07:12:58,013 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:12:58,023 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 13.88it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 17.77it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 17.88it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.68it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 17.51it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 57.08it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 60.75it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 63.98it/s]
+2025-07-31 07:12:58,342 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:12:58,342 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:12:58,342 - user_prompt - INFO - No suitable condition found in semantic search
+2025-07-31 07:12:58,342 - llm_clients - INFO - Calling Medical LLM with query: sudden onset weakness on right side with speech difficulty
+2025-07-31 07:13:09,255 - llm_clients - INFO - Raw LLM Response: Cerebrovascular Accident (CVA), or Acute Ischemic Stroke (specifically, with right hemiparesis and aphasia) 
+
+- This diagnosis represents the most likely condition given the sudden onset of right-sided weakness (hemiparesis) and speech difficulty (aphasia), which are classic symptoms of an ischemic stroke affecting the dominant hemisphere (assuming the patient is right-handed).
+
+Please note that only a qualified physician can confirm a diagnosis after a thorough evaluation, including imaging studies
+2025-07-31 07:13:09,255 - llm_clients - INFO - Query Latency: 10.9129 seconds
+2025-07-31 07:13:09,255 - llm_clients - INFO - Extracted Condition: Cerebrovascular Accident (CVA), or Acute Ischemic Stroke (specifically, with right hemiparesis and aphasia)
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  8.55it/s]
+2025-07-31 07:13:09,844 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 22.223s
+   ⚠️  Test PARTIAL - ⚠️ Level 5 != expected 2. ⚠️ Condition 'generic medical query' != expected ['acute stroke', 'cerebrovascular accident']. 
+
+🔍 level3_001: Level 3: Generic medical terms requiring semantic search
+Query: 'emergency management of cardiovascular crisis'
+Expected Level: 3
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:13:09,854 - llm_clients - INFO - Calling Medical LLM with query: emergency management of cardiovascular crisis
+2025-07-31 07:13:20,094 - llm_clients - INFO - Raw LLM Response: Cardiac Arrest (or, in context of crisis not yet arrest: Acute Cardiogenic Emergency, e.g., STEMI) 
+
+- Note: As a text-based AI assistant, not a clinician, I don't provide medical advice. The term given here represents the most critical cardiovascular crisis requiring immediate emergency intervention. Cardiac arrest implies the heart has stopped pumping, while acute cardiogenic emergency (e.g., ST-elevation myocardial infarction, or STEMI) signifies severe heart
+2025-07-31 07:13:20,095 - llm_clients - INFO - Query Latency: 10.2402 seconds
+2025-07-31 07:13:20,095 - llm_clients - INFO - Extracted Condition: Cardiac Arrest (or, in context of crisis not yet arrest: Acute Cardiogenic Emergency, e.g., STEMI)
+2025-07-31 07:13:20,095 - user_prompt - INFO - Starting semantic search fallback for query: 'emergency management of cardiovascular crisis'
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 15.11it/s]
+2025-07-31 07:13:20,681 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:13:20,713 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 14.75it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 59.28it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 56.29it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.79it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 65.12it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.44it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.88it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.20it/s]
+2025-07-31 07:13:20,905 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:13:20,905 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:13:20,905 - user_prompt - INFO - No suitable condition found in semantic search
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 15.96it/s]
+2025-07-31 07:13:21,492 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 11.647s
+   ⚠️  Test PARTIAL - ⚠️ Level 5 != expected 3. ⚠️ Condition 'generic medical query' != expected []. 
+
+🔍 level3_002: Level 3: Medical terminology requiring semantic fallback
+Query: 'urgent neurological intervention protocols'
+Expected Level: 3
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:13:21,501 - llm_clients - INFO - Calling Medical LLM with query: urgent neurological intervention protocols
+2025-07-31 07:13:30,536 - llm_clients - INFO - Raw LLM Response: The most representative condition: Acute Ischemic Stroke (requiring urgent neurointervention, such as thrombectomy)
+
+Explanation: The phrase "urgent neurological intervention protocols" typically refers to time-critical situations in neurology, and among these, acute ischemic stroke is a prime example. Acute ischemic stroke necessitates rapid evaluation and intervention, including thrombectomy, to restore blood flow and minimize brain damage. This condition demands urgent action due to its narrow therapeutic window, typically within 
+2025-07-31 07:13:30,537 - llm_clients - INFO - Query Latency: 9.0352 seconds
+2025-07-31 07:13:30,537 - llm_clients - INFO - Extracted Condition: The most representative condition: Acute Ischemic Stroke (requiring urgent neurointervention, such as thrombectomy)
+2025-07-31 07:13:30,537 - user_prompt - INFO - Starting semantic search fallback for query: 'urgent neurological intervention protocols'
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  7.94it/s]
+2025-07-31 07:13:31,115 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:13:31,123 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 13.96it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 46.55it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 47.09it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 58.23it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 48.16it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 65.05it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 60.42it/s]
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 63.08it/s]
+2025-07-31 07:13:31,334 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:13:31,334 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:13:31,334 - user_prompt - INFO - No suitable condition found in semantic search
+Batches: 100%|██████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 16.31it/s]
+2025-07-31 07:13:31,889 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 10.398s
+   ⚠️  Test PARTIAL - ⚠️ Level 5 != expected 3. ⚠️ Condition 'generic medical query' != expected []. 
+
+🔍 level4a_001: Level 4a: Non-medical query should be rejected
+Query: 'how to cook pasta properly?'
+Expected Level: 4
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:13:31,899 - llm_clients - INFO - Calling Medical LLM with query: how to cook pasta properly?
+2025-07-31 07:13:41,038 - llm_clients - INFO - Raw LLM Response: As a medical assistant, I do not address cooking techniques, only medical conditions. However, for context (not advice): This query doesn't represent a medical condition; it's about culinary practice. In this case, "properly" cooking pasta typically means achieving al dente texture (not overly soft) by boiling in adequately salted water for the recommended time on the package, then draining well. This is unrelated to any health condition unless discussing, hypothetically, gastrointestinal tolerance in specific patients (e
+2025-07-31 07:13:41,038 - llm_clients - INFO - Query Latency: 9.1386 seconds
+2025-07-31 07:13:41,038 - llm_clients - INFO - Extracted Condition: As a medical assistant, I do not address cooking techniques, only medical conditions. However, for context (not advice): This query doesn't represent a medical condition; it's about culinary practice. In this case, "properly" cooking pasta typically means achieving al dente texture (not overly soft) by boiling in adequately salted water for the recommended time on the package, then draining well. This is unrelated to any health condition unless discussing, hypothetically, gastrointestinal tolerance in specific patients (e
+2025-07-31 07:13:41,038 - user_prompt - INFO - Starting semantic search fallback for query: 'how to cook pasta properly?'
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  2.02it/s]
+2025-07-31 07:13:42,156 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:13:42,165 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  9.34it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 52.88it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 55.97it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 55.95it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 54.63it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 59.07it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 57.84it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 60.43it/s]
+2025-07-31 07:13:42,407 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:13:42,407 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:13:42,407 - user_prompt - INFO - No suitable condition found in semantic search
+2025-07-31 07:13:42,407 - llm_clients - INFO - Calling Medical LLM with query: how to cook pasta properly?
+2025-07-31 07:13:51,634 - llm_clients - INFO - Raw LLM Response: As a medical assistant, I don't address cooking techniques, but for context (not medical advice): In terms of relevance to health, the key aspect here isn't "proper" cooking per se, but rather avoiding overcooking that can reduce nutrient content. For whole-grain pasta, aim for al dente texture (firm, not mushy) to preserve fiber and other nutrients. However, this query is not about a medical condition but a culinary practice.
+ Representative (non-medical) term
+2025-07-31 07:13:51,634 - llm_clients - INFO - Query Latency: 9.2269 seconds
+2025-07-31 07:13:51,634 - llm_clients - INFO - Extracted Condition: As a medical assistant, I don't address cooking techniques, but for context (not medical advice): In terms of relevance to health, the key aspect here isn't "proper" cooking per se, but rather avoiding overcooking that can reduce nutrient content. For whole-grain pasta, aim for al dente texture (firm, not mushy) to preserve fiber and other nutrients. However, this query is not about a medical condition but a culinary practice.
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  2.02it/s]
+2025-07-31 07:13:52,790 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 20.900s
+   ⚠️  Test PARTIAL - ⚠️ Level 5 != expected 4. ⚠️ Query should have been rejected. 
+
+🔍 level4a_002: Level 4a: Technology query should be rejected
+Query: 'best programming language to learn in 2025'
+Expected Level: 4
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:13:52,799 - llm_clients - INFO - Calling Medical LLM with query: best programming language to learn in 2025
+2025-07-31 07:14:02,339 - llm_clients - INFO - Raw LLM Response: As a medical assistant, I do not address technology or education preferences like "best programming language" (which is non-medical context); however, for clarity, this query is outside my biomedical scope. In 2025 or any current year, the choice of "best" programming language is subjective and depends on industry trends, personal goals, and specific use cases (e.g., web development, mobile apps, or data science). Popular choices include Python, JavaScript, and Java, but it's crucial
+2025-07-31 07:14:02,339 - llm_clients - INFO - Query Latency: 9.5400 seconds
+2025-07-31 07:14:02,339 - llm_clients - INFO - Extracted Condition: As a medical assistant, I do not address technology or education preferences like "best programming language" (which is non-medical context); however, for clarity, this query is outside my biomedical scope. In 2025 or any current year, the choice of "best" programming language is subjective and depends on industry trends, personal goals, and specific use cases (e.g., web development, mobile apps, or data science). Popular choices include Python, JavaScript, and Java, but it's crucial
+2025-07-31 07:14:02,339 - user_prompt - INFO - Starting semantic search fallback for query: 'best programming language to learn in 2025'
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  6.45it/s]
+2025-07-31 07:14:02,974 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:14:02,986 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  3.16it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 44.42it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 41.06it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 57.97it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 40.41it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 57.85it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 54.99it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 55.63it/s]
+2025-07-31 07:14:03,457 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:14:03,457 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:14:03,457 - user_prompt - INFO - No suitable condition found in semantic search
+2025-07-31 07:14:03,457 - llm_clients - INFO - Calling Medical LLM with query: best programming language to learn in 2025
+2025-07-31 07:14:13,766 - llm_clients - INFO - Raw LLM Response: As a medical assistant, I don't analyze technology trends or recommend programming languages; however, for clarity's sake (though out of my medical scope), in 2021 (not 2025's future prediction, as I'm bound by current data), popular choices for learning include Python, JavaScript, and Java due to their versatility, wide adoption, and job market demand. Keep in mind this information is not medical advice but rather a layman's interpretation of tech trends.
+
+Representative Condition (not
+2025-07-31 07:14:13,766 - llm_clients - INFO - Query Latency: 10.3088 seconds
+2025-07-31 07:14:13,767 - llm_clients - INFO - Extracted Condition: As a medical assistant, I don't analyze technology trends or recommend programming languages; however, for clarity's sake (though out of my medical scope), in 2021 (not 2025's future prediction, as I'm bound by current data), popular choices for learning include Python, JavaScript, and Java due to their versatility, wide adoption, and job market demand. Keep in mind this information is not medical advice but rather a layman's interpretation of tech trends.
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  2.00it/s]
+2025-07-31 07:14:14,884 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 22.107s
+   ⚠️  Test PARTIAL - ⚠️ Level 5 != expected 4. ⚠️ Query should have been rejected. 
+
+🔍 level4a_003: Level 4a: Weather query should be rejected
+Query: 'weather forecast for tomorrow'
+Expected Level: 4
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:14:14,905 - llm_clients - INFO - Calling Medical LLM with query: weather forecast for tomorrow
+2025-07-31 07:14:24,069 - llm_clients - INFO - Raw LLM Response: As a medical assistant, I do not address weather forecasts; however, for context clarification, this query is unrelated to medical conditions. The requested information here is about meteorology (weather prediction) rather than health or disease. There's no representative medical condition to provide in this case.
+
+If, however, you were referring indirectly to weather-sensitive health conditions (e.g., heat exhaustion, cold-induced asthma exacerbation), the specific condition would depend on the actual weather forecast details (temperature, humidity, etc.)
+2025-07-31 07:14:24,069 - llm_clients - INFO - Query Latency: 9.1634 seconds
+2025-07-31 07:14:24,069 - llm_clients - INFO - Extracted Condition: As a medical assistant, I do not address weather forecasts; however, for context clarification, this query is unrelated to medical conditions. The requested information here is about meteorology (weather prediction) rather than health or disease. There's no representative medical condition to provide in this case.
+2025-07-31 07:14:24,070 - user_prompt - INFO - Starting semantic search fallback for query: 'weather forecast for tomorrow'
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  2.17it/s]
+2025-07-31 07:14:25,222 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:14:25,234 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 11.71it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 50.65it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 55.87it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 50.21it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 21.32it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 54.77it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 53.42it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 56.34it/s]
+2025-07-31 07:14:25,491 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:14:25,491 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:14:25,491 - user_prompt - INFO - No suitable condition found in semantic search
+2025-07-31 07:14:25,491 - llm_clients - INFO - Calling Medical LLM with query: weather forecast for tomorrow
+2025-07-31 07:14:35,356 - llm_clients - INFO - Raw LLM Response: As a medical assistant, I do not address weather forecasts; however, for this context (to maintain representativeness in terms unrelated to diagnosis), the phrase here isn't indicative of a medical condition. Instead, it's about environmental information—specifically, a request for meteorological data (tomorrow's weather). In medical terminology, we wouldn't classify this as a condition, but for representation's sake in a non-medical context, it can be labeled as "meteorological inquiry" or simply
+2025-07-31 07:14:35,356 - llm_clients - INFO - Query Latency: 9.8645 seconds
+2025-07-31 07:14:35,356 - llm_clients - INFO - Extracted Condition: As a medical assistant, I do not address weather forecasts; however, for this context (to maintain representativeness in terms unrelated to diagnosis), the phrase here isn't indicative of a medical condition. Instead, it's about environmental information—specifically, a request for meteorological data (tomorrow's weather). In medical terminology, we wouldn't classify this as a condition, but for representation's sake in a non-medical context, it can be labeled as "meteorological inquiry" or simply
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 11.19it/s]
+2025-07-31 07:14:36,024 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 21.128s
+   ⚠️  Test PARTIAL - ⚠️ Level 5 != expected 4. ⚠️ Query should have been rejected. 
+
+🔍 level4b_001: Level 4b→5: Obscure medical query passing validation to generic search
+Query: 'rare hematologic malignancy treatment approaches'
+Expected Level: 5
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:14:36,033 - llm_clients - INFO - Calling Medical LLM with query: rare hematologic malignancy treatment approaches
+2025-07-31 07:14:45,301 - llm_clients - INFO - Raw LLM Response: The most representative condition: Myelofibrosis (or, in context of "rare" reference, could be an even less common variant like BCR-ABL1-negative atypical CML or unclassifiable myeloproliferative neoplasm) 
+
+- For myelofibrosis, primary treatment approaches include JAK2 inhibitors (e.g., ruxolitinib), supportive care (transfusions, erythropoiesis-stimulating agents), and allog
+2025-07-31 07:14:45,302 - llm_clients - INFO - Query Latency: 9.2678 seconds
+2025-07-31 07:14:45,302 - llm_clients - INFO - Extracted Condition: The most representative condition: Myelofibrosis (or, in context of "rare" reference, could be an even less common variant like BCR-ABL1-negative atypical CML or unclassifiable myeloproliferative neoplasm)
+2025-07-31 07:14:45,302 - user_prompt - INFO - Starting semantic search fallback for query: 'rare hematologic malignancy treatment approaches'
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  2.00it/s]
+2025-07-31 07:14:46,428 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:14:46,436 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 11.59it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 38.61it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 41.66it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.40it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 41.09it/s]
+Batches: 100%|█████████████████████████████���████████████████████████████████████| 1/1 [00:00<00:00, 60.42it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 58.98it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 66.70it/s]
+2025-07-31 07:14:46,672 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:14:46,672 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:14:46,672 - user_prompt - INFO - No suitable condition found in semantic search
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 54.28it/s]
+2025-07-31 07:14:47,160 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 11.137s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level4b_002: Level 4b→5: Rare condition requiring generic medical search
+Query: 'idiopathic thrombocytopenic purpura management guidelines'
+Expected Level: 5
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:14:47,170 - llm_clients - INFO - Calling Medical LLM with query: idiopathic thrombocytopenic purpura management guidelines
+2025-07-31 07:14:56,483 - llm_clients - INFO - Raw LLM Response: The primary medical condition: Idiopathic Thrombocytopenic Purpura (ITP)
+
+(As a medical assistant, I do not provide advice, but here's the relevant condition with context for a knowledge reference.)
+In this case, the most representative condition is Idiopathic Thrombocytopenic Purpura (ITP), an autoimmune disorder characterized by low platelet count (thrombocytopenia) without identifiable underlying causes. Management guidelines typically involve
+2025-07-31 07:14:56,484 - llm_clients - INFO - Query Latency: 9.3136 seconds
+2025-07-31 07:14:56,484 - llm_clients - INFO - Extracted Condition: The primary medical condition: Idiopathic Thrombocytopenic Purpura (ITP)
+2025-07-31 07:14:56,484 - user_prompt - INFO - Starting semantic search fallback for query: 'idiopathic thrombocytopenic purpura management guidelines'
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 11.14it/s]
+2025-07-31 07:14:57,082 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:14:57,090 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 12.83it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 51.94it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 60.06it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 65.59it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 57.81it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.78it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.76it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.14it/s]
+2025-07-31 07:14:57,296 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:14:57,296 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:14:57,296 - user_prompt - INFO - No suitable condition found in semantic search
+2025-07-31 07:14:57,296 - llm_clients - INFO - Calling Medical LLM with query: idiopathic thrombocytopenic purpura management guidelines
+2025-07-31 07:15:06,621 - llm_clients - INFO - Raw LLM Response: The primary medical condition: Idiopathic Thrombocytopenic Purpura (ITP)
+
+(As a medical assistant, I don't provide advice, but describe the condition and point to standard guidelines. For ITP management, refer to professional sources like the American Society of Hematology [ASH] or National Institutes of Health [NIH].)
+
+Idiopathic Thrombocytopenic Purpura (ITP) is an autoimmune disorder characterized by low platelet count
+2025-07-31 07:15:06,621 - llm_clients - INFO - Query Latency: 9.3245 seconds
+2025-07-31 07:15:06,621 - llm_clients - INFO - Extracted Condition: The primary medical condition: Idiopathic Thrombocytopenic Purpura (ITP)
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 11.12it/s]
+2025-07-31 07:15:07,391 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 20.228s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level4b_003: Level 4b→5: Rare emergency condition → generic search
+Query: 'necrotizing fasciitis surgical intervention protocols'
+Expected Level: 5
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:15:07,398 - llm_clients - INFO - Calling Medical LLM with query: necrotizing fasciitis surgical intervention protocols
+2025-07-31 07:15:16,625 - llm_clients - INFO - Raw LLM Response: The primary medical condition: Necrotizing Fasciitis
+
+In this context, the key condition is Necrotizing Fasciitis, a severe bacterial infection characterized by rapid destruction of subcutaneous tissue and fascia. The term provided, "surgical intervention protocols," refers to the treatment approach rather than a distinct medical condition. However, for clarity in this answer, I'll address it as it pertains to managing Necrotizing Fasciitis.
+
+In Necrotizing Fasciitis, surgical
+2025-07-31 07:15:16,625 - llm_clients - INFO - Query Latency: 9.2271 seconds
+2025-07-31 07:15:16,625 - llm_clients - INFO - Extracted Condition: The primary medical condition: Necrotizing Fasciitis
+2025-07-31 07:15:16,625 - user_prompt - INFO - Starting semantic search fallback for query: 'necrotizing fasciitis surgical intervention protocols'
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 10.01it/s]
+2025-07-31 07:15:17,212 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:15:17,222 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 12.01it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 45.04it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 44.57it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 57.92it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 48.15it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.28it/s]
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 60.83it/s]
+Batches: 100%|███���██████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.38it/s]
+2025-07-31 07:15:17,449 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:15:17,449 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:15:17,449 - user_prompt - INFO - No suitable condition found in semantic search
+2025-07-31 07:15:17,449 - llm_clients - INFO - Calling Medical LLM with query: necrotizing fasciitis surgical intervention protocols
+2025-07-31 07:15:24,511 - llm_clients - INFO - Raw LLM Response: The most representative condition: Necrotizing Fasciitis
+
+(As a medical assistant, I do not provide advice, only identify conditions. For necrotizing fasciitis, surgical intervention typically involves aggressive debridement—removing dead tissue—and may require repeated procedures until healthy margins are achieved. This is accompanied by supportive care and antibiotics.)
+
+
+2025-07-31 07:15:24,511 - llm_clients - INFO - Query Latency: 7.0619 seconds
+2025-07-31 07:15:24,511 - llm_clients - INFO - Extracted Condition: The most representative condition: Necrotizing Fasciitis
+Batches: 100%|██████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 13.83it/s]
+2025-07-31 07:15:25,078 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 17.692s
+   🎉 Test PASSED - Expected behavior achieved
+
+================================================================================
+📊 MULTILEVEL FALLBACK TEST REPORT
+================================================================================
+🕐 Execution Summary:
+   Total duration: 187.465s
+   Average per test: 14.420s
+
+📈 Test Results:
+   Total tests: 13
+   Passed: 7 ✅
+   Partial: 6 ⚠️
+   Failed: 6 ❌
+   Success rate: 53.8%
+
+🎯 Level Distribution Analysis:
+   Level 1 (Predefined Mapping): 4 tests, avg 5.752s
+   Level 5 (Generic Search): 9 tests, avg 17.495s
+
+📋 Category Analysis:
+   level1_predefined: 3/3 (100.0%)
+   level2_llm: 1/2 (50.0%)
+   level3_semantic: 0/2 (0.0%)
+   level4a_rejection: 0/3 (0.0%)
+   level4b_to_5: 3/3 (100.0%)
+
+📝 Detailed Test Results:
+
+   level1_001: ✅ PASS
+      Query: 'acute myocardial infarction treatment'
+      Expected Level: 1
+      Detected Level: 1
+      Condition: acute myocardial infarction
+      Time: 0.000s
+      Validation: ✅ Level 1 as expected. ✅ Condition 'acute myocardial infarction' matches expected. 
+
+   level1_002: ✅ PASS
+      Query: 'how to manage acute stroke?'
+      Expected Level: 1
+      Detected Level: 1
+      Condition: acute stroke
+      Time: 0.000s
+      Validation: ✅ Level 1 as expected. ✅ Condition 'acute stroke' matches expected. 
+
+   level1_003: ✅ PASS
+      Query: 'pulmonary embolism emergency protocol'
+      Expected Level: 1
+      Detected Level: 1
+      Condition: pulmonary embolism
+      Time: 0.000s
+      Validation: ✅ Level 1 as expected. ✅ Condition 'pulmonary embolism' matches expected. 
+
+   level2_001: ✅ PASS
+      Query: 'patient with severe crushing chest pain radiating to left arm'
+      Expected Level: 2
+      Detected Level: 1
+      Condition: acute myocardial infarction
+      Time: 23.008s
+      Validation: ⚠️ Level 1 != expected 2. ✅ Condition 'acute myocardial infarction' matches expected. 
+
+   level2_002: ⚠️ PARTIAL
+      Query: 'sudden onset weakness on right side with speech difficulty'
+      Expected Level: 2
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 22.223s
+      Validation: ⚠️ Level 5 != expected 2. ⚠️ Condition 'generic medical query' != expected ['acute stroke', 'cerebrovascular accident']. 
+
+   level3_001: ⚠️ PARTIAL
+      Query: 'emergency management of cardiovascular crisis'
+      Expected Level: 3
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 11.647s
+      Validation: ⚠️ Level 5 != expected 3. ⚠️ Condition 'generic medical query' != expected []. 
+
+   level3_002: ⚠️ PARTIAL
+      Query: 'urgent neurological intervention protocols'
+      Expected Level: 3
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 10.398s
+      Validation: ⚠️ Level 5 != expected 3. ⚠️ Condition 'generic medical query' != expected []. 
+
+   level4a_001: ⚠️ PARTIAL
+      Query: 'how to cook pasta properly?'
+      Expected Level: 4
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 20.900s
+      Validation: ⚠️ Level 5 != expected 4. ⚠️ Query should have been rejected. 
+
+   level4a_002: ⚠️ PARTIAL
+      Query: 'best programming language to learn in 2025'
+      Expected Level: 4
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 22.107s
+      Validation: ⚠️ Level 5 != expected 4. ⚠️ Query should have been rejected. 
+
+   level4a_003: ⚠️ PARTIAL
+      Query: 'weather forecast for tomorrow'
+      Expected Level: 4
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 21.128s
+      Validation: ⚠️ Level 5 != expected 4. ⚠️ Query should have been rejected. 
+
+   level4b_001: ✅ PASS
+      Query: 'rare hematologic malignancy treatment approaches'
+      Expected Level: 5
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 11.137s
+      Validation: ✅ Level 5 as expected. ✅ Generic medical search triggered. 
+
+   level4b_002: ✅ PASS
+      Query: 'idiopathic thrombocytopenic purpura management guidelines'
+      Expected Level: 5
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 20.228s
+      Validation: ✅ Level 5 as expected. ✅ Generic medical search triggered. 
+
+   level4b_003: ✅ PASS
+      Query: 'necrotizing fasciitis surgical intervention protocols'
+      Expected Level: 5
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 17.692s
+      Validation: ✅ Level 5 as expected. ✅ Generic medical search triggered. 

tests/result_of_test_multlevel_fallback_validation_revised.md

@@ -0,0 +1,534 @@
+🏥 OnCall.ai Multilevel Fallback Validation Test
+============================================================
+🔧 Initializing Components for Multilevel Fallback Test...
+------------------------------------------------------------
+1. Initializing Llama3-Med42-70B Client...
+2025-07-31 07:51:06,059 - llm_clients - INFO - Medical LLM client initialized with model: m42-health/Llama3-Med42-70B
+2025-07-31 07:51:06,059 - llm_clients - WARNING - Medical LLM Model: Research tool only. Not for professional medical diagnosis.
+   ✅ LLM client initialized
+2. Initializing Retrieval System...
+2025-07-31 07:51:06,059 - retrieval - INFO - Initializing retrieval system...
+2025-07-31 07:51:06,073 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-07-31 07:51:06,073 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: NeuML/pubmedbert-base-embeddings
+2025-07-31 07:51:09,264 - retrieval - INFO - Embedding model loaded successfully
+2025-07-31 07:51:10,711 - retrieval - INFO - Chunks loaded successfully
+2025-07-31 07:51:10,824 - retrieval - INFO - Embeddings loaded successfully
+2025-07-31 07:51:10,825 - retrieval - INFO - Loaded existing emergency index
+2025-07-31 07:51:10,826 - retrieval - INFO - Loaded existing treatment index
+2025-07-31 07:51:10,826 - retrieval - INFO - Retrieval system initialized successfully
+   ✅ Retrieval system initialized
+3. Initializing User Prompt Processor...
+2025-07-31 07:51:10,826 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-07-31 07:51:10,826 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: NeuML/pubmedbert-base-embeddings
+2025-07-31 07:51:12,702 - user_prompt - INFO - UserPromptProcessor initialized
+   ✅ User prompt processor initialized
+
+🎉 All components initialized successfully!
+
+🚀 Starting Multilevel Fallback Test Suite
+Total test cases: 13
+Test started at: 2025-07-31 07:51:06
+================================================================================
+
+🔍 level1_001: Level 1: Direct predefined condition match
+Query: 'acute myocardial infarction treatment'
+Expected Level: 1
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:51:12,702 - user_prompt - INFO - Matched predefined condition: acute myocardial infarction
+   ✅ Detected Level: 1
+   Condition: acute myocardial infarction
+   Emergency Keywords: MI|chest pain|cardiac arrest
+   Treatment Keywords: aspirin|nitroglycerin|thrombolytic|PCI
+   Execution Time: 0.000s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level1_002: Level 1: Predefined stroke condition
+Query: 'how to manage acute stroke?'
+Expected Level: 1
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:51:12,702 - user_prompt - INFO - Matched predefined condition: acute stroke
+   ✅ Detected Level: 1
+   Condition: acute stroke
+   Emergency Keywords: stroke|neurological deficit|sudden weakness
+   Treatment Keywords: tPA|thrombolysis|stroke unit care
+   Execution Time: 0.000s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level1_003: Level 1: Predefined PE condition
+Query: 'pulmonary embolism emergency protocol'
+Expected Level: 1
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:51:12,702 - user_prompt - INFO - Matched predefined condition: pulmonary embolism
+   ✅ Detected Level: 1
+   Condition: pulmonary embolism
+   Emergency Keywords: chest pain|shortness of breath|sudden dyspnea
+   Treatment Keywords: anticoagulation|heparin|embolectomy
+   Execution Time: 0.000s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level2_001: Level 2: Symptom-based query requiring LLM analysis
+Query: 'patient with severe crushing chest pain radiating to left arm'
+Expected Level: 2
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:51:12,702 - llm_clients - INFO - Calling Medical LLM with query: patient with severe crushing chest pain radiating to left arm
+2025-07-31 07:51:55,277 - llm_clients - INFO - Raw LLM Response: Medical: "Acute Myocardial Infarction" (Heart Attack)
+Explanation: The described symptoms of severe crushing chest pain radiating to the left arm are highly indicative of an acute myocardial infarction, commonly known as a heart attack. This is a medical emergency caused by blockage of coronary arteries, disrupting blood supply to the heart muscle.
+
+(Not providing advice, just categorizing the condition)
+2025-07-31 07:51:55,278 - llm_clients - INFO - Query Latency: 42.5747 seconds
+2025-07-31 07:51:55,278 - llm_clients - INFO - Extracted Condition: acute myocardial infarction
+   ✅ Detected Level: 1
+   Condition: acute myocardial infarction
+   Emergency Keywords: MI|chest pain|cardiac arrest
+   Treatment Keywords: aspirin|nitroglycerin|thrombolytic|PCI
+   Execution Time: 42.576s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level2_002: Level 2: Neurological symptoms requiring LLM
+Query: 'sudden onset weakness on right side with speech difficulty'
+Expected Level: 2
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:51:55,279 - llm_clients - INFO - Calling Medical LLM with query: sudden onset weakness on right side with speech difficulty
+2025-07-31 07:52:06,165 - llm_clients - INFO - Raw LLM Response: Medical: "Acute Ischemic Stroke" (or Cerebrovascular Accident, specifically involving right hemispheric damage causing contralateral weakness and speech impairment)
+
+Explanation: The symptoms described - sudden onset weakness on the right side (implying left brain hemisphere involvement due to contralateral motor control) and speech difficulty - are classic indicators of an acute ischemic stroke. This condition occurs when blood flow to a region of the brain is blocked, depriving it of oxygen and nutrients,
+2025-07-31 07:52:06,165 - llm_clients - INFO - Query Latency: 10.8864 seconds
+2025-07-31 07:52:06,165 - llm_clients - INFO - Extracted Condition: Medical: "Acute Ischemic Stroke" (or Cerebrovascular Accident, specifically involving right hemispheric damage causing contralateral weakness and speech impairment)
+2025-07-31 07:52:06,166 - user_prompt - INFO - Starting semantic search fallback for query: 'sudden onset weakness on right side with speech difficulty'
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  1.61it/s]
+2025-07-31 07:52:07,568 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:52:07,575 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 14.70it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 17.71it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 17.64it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.46it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 16.59it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 57.61it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 57.26it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 56.86it/s]
+2025-07-31 07:52:07,896 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:52:07,896 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:52:07,896 - user_prompt - INFO - No suitable condition found in semantic search
+2025-07-31 07:52:07,897 - llm_clients - INFO - Calling Medical LLM with query: sudden onset weakness on right side with speech difficulty
+2025-07-31 07:52:16,923 - llm_clients - INFO - Raw LLM Response: Medical: "Cerebrovascular Accident (CVA) - Ischemic Stroke" (or simply "Ischemic Stroke" for brevity, as it's the most specific diagnosis here)
+  - Explanation: The symptoms described, sudden right-sided weakness and speech difficulty, are classic indicators of an ischemic stroke, which occurs when blood flow to the brain is blocked by a clot or narrowed blood vessels.
+
+Note: While hemorrhagic stroke is another type of CVA, the given symptoms
+2025-07-31 07:52:16,923 - llm_clients - INFO - Query Latency: 9.0264 seconds
+2025-07-31 07:52:16,923 - llm_clients - INFO - Extracted Condition: Medical: "Cerebrovascular Accident (CVA) - Ischemic Stroke" (or simply "Ischemic Stroke" for brevity, as it's the most specific diagnosis here)
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  1.89it/s]
+2025-07-31 07:52:17,964 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 22.751s
+   ⚠️  Test PARTIAL - ⚠️ Level 5 != expected 2. ⚠️ Condition 'generic medical query' != expected ['acute stroke', 'cerebrovascular accident']. 
+
+🔍 level3_001: Level 3: Generic medical terms requiring semantic search
+Query: 'emergency management of cardiovascular crisis'
+Expected Level: 3
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:52:18,030 - llm_clients - INFO - Calling Medical LLM with query: emergency management of cardiovascular crisis
+2025-07-31 07:52:27,145 - llm_clients - INFO - Raw LLM Response: Medical: "Cardiovascular crisis" in this context (emergency management) is best represented by "Acute Myocardial Infarction (AMI)" or "ST-Elevation Myocardial Infarction (STEMI)," as both terms describe severe, time-critical cardiac events requiring immediate intervention. However, if considering a broader "cardiovascular crisis" that's not limited to infarction, "Cardiogenic Shock" might also be applicable, as it represents a severe, life
+2025-07-31 07:52:27,145 - llm_clients - INFO - Query Latency: 9.1143 seconds
+2025-07-31 07:52:27,145 - llm_clients - INFO - Extracted Condition: acute myocardial infarction
+   ✅ Detected Level: 1
+   Condition: acute myocardial infarction
+   Emergency Keywords: MI|chest pain|cardiac arrest
+   Treatment Keywords: aspirin|nitroglycerin|thrombolytic|PCI
+   Execution Time: 9.115s
+   ⚠️  Test PARTIAL - ⚠️ Level 1 != expected 3. ⚠️ Condition 'acute myocardial infarction' != expected []. 
+
+🔍 level3_002: Level 3: Medical terminology requiring semantic fallback
+Query: 'urgent neurological intervention protocols'
+Expected Level: 3
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:52:27,145 - llm_clients - INFO - Calling Medical LLM with query: urgent neurological intervention protocols
+2025-07-31 07:52:37,615 - llm_clients - INFO - Raw LLM Response: Medical: "Acute Ischemic Stroke" (representing a condition requiring urgent neurological intervention, specifically thrombectomy or thrombolysis protocols)
+
+Explanation: Acute ischemic stroke necessitates rapid medical response, as timely interventions like mechanical thrombectomy or intravenous thrombolysis can significantly improve patient outcomes. The term "urgent neurological intervention protocols" in this context likely refers to these treatments for stroke, making "Acute Ischemic Stroke" the most representative medical condition.
+2025-07-31 07:52:37,615 - llm_clients - INFO - Query Latency: 10.4695 seconds
+2025-07-31 07:52:37,615 - llm_clients - INFO - Extracted Condition: Medical: "Acute Ischemic Stroke" (representing a condition requiring urgent neurological intervention, specifically thrombectomy or thrombolysis protocols)
+2025-07-31 07:52:37,616 - user_prompt - INFO - Starting semantic search fallback for query: 'urgent neurological intervention protocols'
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  3.45it/s]
+2025-07-31 07:52:38,539 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:52:38,549 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 13.55it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 49.53it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 52.73it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 64.13it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 51.36it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.40it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 58.10it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.29it/s]
+2025-07-31 07:52:38,759 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:52:38,759 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:52:38,759 - user_prompt - INFO - No suitable condition found in semantic search
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 16.09it/s]
+2025-07-31 07:52:39,345 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 12.249s
+   ⚠️  Test PARTIAL - ⚠️ Level 5 != expected 3. ⚠️ Condition 'generic medical query' != expected []. 
+
+🔍 level4a_001: Level 4a: Non-medical query should be rejected
+Query: 'how to cook pasta properly?'
+Expected Level: 4
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:52:39,395 - llm_clients - INFO - Calling Medical LLM with query: how to cook pasta properly?
+2025-07-31 07:52:45,753 - llm_clients - INFO - Raw LLM Response: NON_MEDICAL_QUERY. This inquiry is about culinary technique (cooking pasta) and not related to medical conditions or health issues. It does not represent a medical topic for diagnosis or advice. Instead, it's a question of food preparation, typically addressed in cookbooks or culinary resources.
+2025-07-31 07:52:45,753 - llm_clients - INFO - Query Latency: 6.3575 seconds
+2025-07-31 07:52:45,753 - llm_clients - INFO - Extracted Condition: 
+2025-07-31 07:52:45,753 - user_prompt - INFO - Starting semantic search fallback for query: 'how to cook pasta properly?'
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  1.47it/s]
+2025-07-31 07:52:47,084 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:52:47,091 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 14.54it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 46.74it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 46.14it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 57.37it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 43.26it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 60.53it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 58.35it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 60.17it/s]
+2025-07-31 07:52:47,305 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:52:47,305 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:52:47,305 - user_prompt - INFO - No suitable condition found in semantic search
+2025-07-31 07:52:47,305 - llm_clients - INFO - Calling Medical LLM with query: how to cook pasta properly?
+2025-07-31 07:52:53,999 - llm_clients - INFO - Raw LLM Response: NON_MEDICAL_QUERY. This inquiry is about culinary technique (cooking pasta) and not related to any medical condition or health issue. It involves instructions on food preparation rather than addressing a disease, symptom, or medical concern.
+2025-07-31 07:52:53,999 - llm_clients - INFO - Query Latency: 6.6933 seconds
+2025-07-31 07:52:53,999 - llm_clients - INFO - Extracted Condition: 
+   ✅ Detected Level: 4
+   Condition: None
+   Emergency Keywords: None
+   Treatment Keywords: None
+   Execution Time: 14.604s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level4a_002: Level 4a: Technology query should be rejected
+Query: 'best programming language to learn in 2025'
+Expected Level: 4
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:52:54,000 - llm_clients - INFO - Calling Medical LLM with query: best programming language to learn in 2025
+2025-07-31 07:53:00,100 - llm_clients - INFO - Raw LLM Response: NON_MEDICAL_QUERY. This inquiry is about technology (specifically, programming languages) and their future relevance, rather than a medical condition or health topic. It doesn't pertain to diagnosis, treatment, or any medical aspect. Therefore, it's not a medical query.
+2025-07-31 07:53:00,100 - llm_clients - INFO - Query Latency: 6.1004 seconds
+2025-07-31 07:53:00,100 - llm_clients - INFO - Extracted Condition: 
+2025-07-31 07:53:00,100 - user_prompt - INFO - Starting semantic search fallback for query: 'best programming language to learn in 2025'
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  2.94it/s]
+2025-07-31 07:53:00,968 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:53:01,048 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 13.26it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 56.37it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 56.33it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.62it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 56.23it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 58.59it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 58.53it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 59.69it/s]
+2025-07-31 07:53:01,255 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:53:01,255 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:53:01,255 - user_prompt - INFO - No suitable condition found in semantic search
+2025-07-31 07:53:01,256 - llm_clients - INFO - Calling Medical LLM with query: best programming language to learn in 2025
+2025-07-31 07:53:06,397 - llm_clients - INFO - Raw LLM Response: NON_MEDICAL_QUERY. This inquiry is about selecting a programming language for future learning (in 2025) and has no relation to medical conditions or healthcare. It falls under the domain of computer science and technology education.
+2025-07-31 07:53:06,397 - llm_clients - INFO - Query Latency: 5.1410 seconds
+2025-07-31 07:53:06,397 - llm_clients - INFO - Extracted Condition: 
+   ✅ Detected Level: 4
+   Condition: None
+   Emergency Keywords: None
+   Treatment Keywords: None
+   Execution Time: 12.397s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level4a_003: Level 4a: Weather query should be rejected
+Query: 'weather forecast for tomorrow'
+Expected Level: 4
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:53:06,397 - llm_clients - INFO - Calling Medical LLM with query: weather forecast for tomorrow
+2025-07-31 07:53:11,119 - llm_clients - INFO - Raw LLM Response: NON_MEDICAL_QUERY. This inquiry is about meteorological information (weather prediction) and not related to any medical condition or health topic. It falls under environmental or general information, not medicine.
+2025-07-31 07:53:11,120 - llm_clients - INFO - Query Latency: 4.7219 seconds
+2025-07-31 07:53:11,120 - llm_clients - INFO - Extracted Condition: 
+2025-07-31 07:53:11,120 - user_prompt - INFO - Starting semantic search fallback for query: 'weather forecast for tomorrow'
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  2.01it/s]
+2025-07-31 07:53:12,200 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:53:12,209 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 13.36it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 51.03it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 53.14it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 63.88it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 55.51it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.11it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.37it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 65.13it/s]
+2025-07-31 07:53:12,415 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:53:12,415 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:53:12,415 - user_prompt - INFO - No suitable condition found in semantic search
+2025-07-31 07:53:12,415 - llm_clients - INFO - Calling Medical LLM with query: weather forecast for tomorrow
+2025-07-31 07:53:17,281 - llm_clients - INFO - Raw LLM Response: NON_MEDICAL_QUERY. This inquiry is about meteorology (predicting weather conditions) and not related to medical conditions or health issues. It doesn't involve symptoms, diagnoses, or any aspect of healthcare.
+2025-07-31 07:53:17,281 - llm_clients - INFO - Query Latency: 4.8653 seconds
+2025-07-31 07:53:17,281 - llm_clients - INFO - Extracted Condition: 
+   ✅ Detected Level: 4
+   Condition: None
+   Emergency Keywords: None
+   Treatment Keywords: None
+   Execution Time: 10.884s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level4b_001: Level 4b→5: Obscure medical query passing validation to generic search
+Query: 'rare hematologic malignancy treatment approaches'
+Expected Level: 5
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:53:17,282 - llm_clients - INFO - Calling Medical LLM with query: rare hematologic malignancy treatment approaches
+2025-07-31 07:53:26,329 - llm_clients - INFO - Raw LLM Response: Medical: "rare hematologic malignancy treatment approaches" → "Targeted Therapy for Agnogenic Myeloid Metaplasia (or currently, 'Agnogenic/Idiopathic: Myelofibrosis' in modern classification, as part of CMML-excluded rare myeloproliferative neoplasms) or, alternatively, 'Chimeric Antigen Receptor T-Cell therapy (CAR-T) for rare B-cell lymphomas like Primary Mediastinal
+2025-07-31 07:53:26,329 - llm_clients - INFO - Query Latency: 9.0470 seconds
+2025-07-31 07:53:26,331 - llm_clients - INFO - Extracted Condition: Medical: "rare hematologic malignancy treatment approaches" → "Targeted Therapy for Agnogenic Myeloid Metaplasia (or currently, 'Agnogenic/Idiopathic: Myelofibrosis' in modern classification, as part of CMML-excluded rare myeloproliferative neoplasms) or, alternatively, 'Chimeric Antigen Receptor T-Cell therapy (CAR-T) for rare B-cell lymphomas like Primary Mediastinal
+2025-07-31 07:53:26,331 - user_prompt - INFO - Starting semantic search fallback for query: 'rare hematologic malignancy treatment approaches'
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 11.05it/s]
+2025-07-31 07:53:26,871 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:53:26,880 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 12.32it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 57.77it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 60.97it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.97it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 59.97it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 58.87it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 58.11it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.43it/s]
+2025-07-31 07:53:27,089 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:53:27,089 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:53:27,089 - user_prompt - INFO - No suitable condition found in semantic search
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 44.43it/s]
+2025-07-31 07:53:27,626 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 10.356s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level4b_002: Level 4b→5: Rare condition requiring generic medical search
+Query: 'idiopathic thrombocytopenic purpura management guidelines'
+Expected Level: 5
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:53:27,638 - llm_clients - INFO - Calling Medical LLM with query: idiopathic thrombocytopenic purpura management guidelines
+2025-07-31 07:53:36,704 - llm_clients - INFO - Raw LLM Response: Medical: "Idiopathic Thrombocytopenic Purpura (ITP) Management" → "ITP Treatment Protocols" (referring to guidelines for managing this autoimmune platelet disorder, which include corticosteroids, IVIG, thrombopoietin receptor agonists, or splenectomy in certain cases)
+
+Explanation: This query is medical because it pertains to the guidelines for treating a specific blood disorder, Idiopathic Thrombocytopenic
+2025-07-31 07:53:36,704 - llm_clients - INFO - Query Latency: 9.0658 seconds
+2025-07-31 07:53:36,704 - llm_clients - INFO - Extracted Condition: Medical: "Idiopathic Thrombocytopenic Purpura (ITP) Management" → "ITP Treatment Protocols" (referring to guidelines for managing this autoimmune platelet disorder, which include corticosteroids, IVIG, thrombopoietin receptor agonists, or splenectomy in certain cases)
+2025-07-31 07:53:36,704 - user_prompt - INFO - Starting semantic search fallback for query: 'idiopathic thrombocytopenic purpura management guidelines'
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 10.61it/s]
+2025-07-31 07:53:37,450 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:53:37,459 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 11.72it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 44.43it/s]
+Batches: 100%|███████████████████████��███████████████████████████████████| 1/1 [00:00<00:00, 41.51it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 60.38it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 43.72it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 59.74it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 59.79it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 60.22it/s]
+2025-07-31 07:53:37,691 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:53:37,691 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:53:37,691 - user_prompt - INFO - No suitable condition found in semantic search
+2025-07-31 07:53:37,691 - llm_clients - INFO - Calling Medical LLM with query: idiopathic thrombocytopenic purpura management guidelines
+2025-07-31 07:53:47,836 - llm_clients - INFO - Raw LLM Response: Medical: "Idiopathic Thrombocytopenic Purpura (ITP) Management" → "ITP Treatment Protocols" (referring to guidelines for therapy in this autoimmune platelet disorder, which may include corticosteroids, intravenous immunoglobulin, or thrombopoietin receptor agonists, among other strategies).
+
+Explanation: This query is medical because it discusses guidelines for managing a specific blood disorder, idiopathic thrombocytopenic purpura
+2025-07-31 07:53:47,836 - llm_clients - INFO - Query Latency: 10.1445 seconds
+2025-07-31 07:53:47,836 - llm_clients - INFO - Extracted Condition: Medical: "Idiopathic Thrombocytopenic Purpura (ITP) Management" → "ITP Treatment Protocols" (referring to guidelines for therapy in this autoimmune platelet disorder, which may include corticosteroids, intravenous immunoglobulin, or thrombopoietin receptor agonists, among other strategies).
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  2.06it/s]
+2025-07-31 07:53:48,812 - retrieval - INFO - Sliding window search: Found 5 results
+   ✅ Detected Level: 5
+   Condition: generic medical query
+   Emergency Keywords: medical|emergency
+   Treatment Keywords: treatment|management
+   Execution Time: 21.183s
+   🎉 Test PASSED - Expected behavior achieved
+
+🔍 level4b_003: Level 4b→5: Rare emergency condition → generic search
+Query: 'necrotizing fasciitis surgical intervention protocols'
+Expected Level: 5
+----------------------------------------------------------------------
+🎯 Executing multilevel fallback...
+2025-07-31 07:53:48,821 - llm_clients - INFO - Calling Medical LLM with query: necrotizing fasciitis surgical intervention protocols
+2025-07-31 07:53:57,799 - llm_clients - INFO - Raw LLM Response: Medical: "Necrotizing Fasciitis" - In this context, the primary medical condition is Necrotizing Fasciitis, a severe soft tissue infection characterized by rapid progression and tissue death. The phrase "surgical intervention protocols" refers to the medical procedures and guidelines for surgically managing this condition, typically involving debridement (removal of dead tissue) and sometimes amputation.
+
+Explanation: This query is medical because it pertains to a specific infectious disease (Necrotizing
+2025-07-31 07:53:57,799 - llm_clients - INFO - Query Latency: 8.9777 seconds
+2025-07-31 07:53:57,800 - llm_clients - INFO - Extracted Condition: Medical: "Necrotizing Fasciitis" - In this context, the primary medical condition is Necrotizing Fasciitis, a severe soft tissue infection characterized by rapid progression and tissue death. The phrase "surgical intervention protocols" refers to the medical procedures and guidelines for surgically managing this condition, typically involving debridement (removal of dead tissue) and sometimes amputation.
+2025-07-31 07:53:57,800 - user_prompt - INFO - Starting semantic search fallback for query: 'necrotizing fasciitis surgical intervention protocols'
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 13.58it/s]
+2025-07-31 07:53:58,405 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 07:53:58,414 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 11.81it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 48.09it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 47.49it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.11it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 46.81it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 60.57it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 59.03it/s]
+Batches: 100%|███████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.29it/s]
+2025-07-31 07:53:58,638 - user_prompt - INFO - Inferred condition: None
+2025-07-31 07:53:58,638 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 07:53:58,638 - user_prompt - INFO - No suitable condition found in semantic search
+2025-07-31 07:53:58,638 - llm_clients - INFO - Calling Medical LLM with query: necrotizing fasciitis surgical intervention protocols
+2025-07-31 07:53:58,758 - llm_clients - ERROR - Medical LLM query error: 402 Client Error: Payment Required for url: https://router.huggingface.co/featherless-ai/v1/chat/completions (Request ID: Root=1-688b8386-259e81a24556b80a163e3d17;5ec89b2d-e0da-4255-90b6-f0c7e9577b38)
+
+You have exceeded your monthly included credits for Inference Providers. Subscribe to PRO to get 20x more monthly included credits.
+2025-07-31 07:53:58,758 - llm_clients - ERROR - Error Type: HfHubHTTPError
+2025-07-31 07:53:58,758 - llm_clients - ERROR - Detailed Error: HfHubHTTPError('402 Client Error: Payment Required for url: https://router.huggingface.co/featherless-ai/v1/chat/completions (Request ID: Root=1-688b8386-259e81a24556b80a163e3d17;5ec89b2d-e0da-4255-90b6-f0c7e9577b38)\n\nYou have exceeded your monthly included credits for Inference Providers. Subscribe to PRO to get 20x more monthly included credits.')
+2025-07-31 07:53:58,758 - llm_clients - ERROR - Query Latency (on error): 0.1196 seconds
+2025-07-31 07:53:58,758 - llm_clients - ERROR - Query that caused error: necrotizing fasciitis surgical intervention protocols
+   ✅ Detected Level: 4
+   Condition: None
+   Emergency Keywords: None
+   Treatment Keywords: None
+   Execution Time: 9.937s
+   ⚠️  Test PARTIAL - ⚠️ Level 4 != expected 5. ⚠️ Should trigger generic medical search. 
+
+================================================================================
+📊 MULTILEVEL FALLBACK TEST REPORT
+================================================================================
+🕐 Execution Summary:
+   Total duration: 172.699s
+   Average per test: 13.285s
+
+📈 Test Results:
+   Total tests: 13
+   Passed: 9 ✅
+   Partial: 4 ⚠️
+   Failed: 4 ❌
+   Success rate: 69.2%
+
+🎯 Level Distribution Analysis:
+   Level 1 (Predefined Mapping): 5 tests, avg 10.338s
+   Level 4 (Validation Rejection): 4 tests, avg 11.956s
+   Level 5 (Generic Search): 4 tests, avg 16.635s
+
+📋 Category Analysis:
+   level1_predefined: 3/3 (100.0%)
+   level2_llm: 1/2 (50.0%)
+   level3_semantic: 0/2 (0.0%)
+   level4a_rejection: 3/3 (100.0%)
+   level4b_to_5: 2/3 (66.7%)
+
+📝 Detailed Test Results:
+
+   level1_001: ✅ PASS
+      Query: 'acute myocardial infarction treatment'
+      Expected Level: 1
+      Detected Level: 1
+      Condition: acute myocardial infarction
+      Time: 0.000s
+      Validation: ✅ Level 1 as expected. ✅ Condition 'acute myocardial infarction' matches expected. 
+
+   level1_002: ✅ PASS
+      Query: 'how to manage acute stroke?'
+      Expected Level: 1
+      Detected Level: 1
+      Condition: acute stroke
+      Time: 0.000s
+      Validation: ✅ Level 1 as expected. ✅ Condition 'acute stroke' matches expected. 
+
+   level1_003: ✅ PASS
+      Query: 'pulmonary embolism emergency protocol'
+      Expected Level: 1
+      Detected Level: 1
+      Condition: pulmonary embolism
+      Time: 0.000s
+      Validation: ✅ Level 1 as expected. ✅ Condition 'pulmonary embolism' matches expected. 
+
+   level2_001: ✅ PASS
+      Query: 'patient with severe crushing chest pain radiating to left arm'
+      Expected Level: 2
+      Detected Level: 1
+      Condition: acute myocardial infarction
+      Time: 42.576s
+      Validation: ⚠️ Level 1 != expected 2. ✅ Condition 'acute myocardial infarction' matches expected. 
+
+   level2_002: ⚠️ PARTIAL
+      Query: 'sudden onset weakness on right side with speech difficulty'
+      Expected Level: 2
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 22.751s
+      Validation: ⚠️ Level 5 != expected 2. ⚠️ Condition 'generic medical query' != expected ['acute stroke', 'cerebrovascular accident']. 
+
+   level3_001: ⚠️ PARTIAL
+      Query: 'emergency management of cardiovascular crisis'
+      Expected Level: 3
+      Detected Level: 1
+      Condition: acute myocardial infarction
+      Time: 9.115s
+      Validation: ⚠️ Level 1 != expected 3. ⚠️ Condition 'acute myocardial infarction' != expected []. 
+
+   level3_002: ⚠️ PARTIAL
+      Query: 'urgent neurological intervention protocols'
+      Expected Level: 3
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 12.249s
+      Validation: ⚠️ Level 5 != expected 3. ⚠️ Condition 'generic medical query' != expected []. 
+
+   level4a_001: ✅ PASS
+      Query: 'how to cook pasta properly?'
+      Expected Level: 4
+      Detected Level: 4
+      Condition: None
+      Time: 14.604s
+      Validation: ✅ Level 4 as expected. ✅ Query correctly rejected. 
+
+   level4a_002: ✅ PASS
+      Query: 'best programming language to learn in 2025'
+      Expected Level: 4
+      Detected Level: 4
+      Condition: None
+      Time: 12.397s
+      Validation: ✅ Level 4 as expected. ✅ Query correctly rejected. 
+
+   level4a_003: ✅ PASS
+      Query: 'weather forecast for tomorrow'
+      Expected Level: 4
+      Detected Level: 4
+      Condition: None
+      Time: 10.884s
+      Validation: ✅ Level 4 as expected. ✅ Query correctly rejected. 
+
+   level4b_001: ✅ PASS
+      Query: 'rare hematologic malignancy treatment approaches'
+      Expected Level: 5
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 10.356s
+      Validation: ✅ Level 5 as expected. ✅ Generic medical search triggered. 
+
+   level4b_002: ✅ PASS
+      Query: 'idiopathic thrombocytopenic purpura management guidelines'
+      Expected Level: 5
+      Detected Level: 5
+      Condition: generic medical query
+      Time: 21.183s
+      Validation: ✅ Level 5 as expected. ✅ Generic medical search triggered. 
+
+   level4b_003: ⚠️ PARTIAL
+      Query: 'necrotizing fasciitis surgical intervention protocols'
+      Expected Level: 5
+      Detected Level: 4
+      Condition: None
+      Time: 9.937s
+      Validation: ⚠️ Level 4 != expected 5. ⚠️ Should trigger generic medical search. 

tests/result_of_test_userinput_userprompt_medical_condition_llm.md

@@ -0,0 +1,381 @@
+🏥 OnCall.ai Medical Query Processing Pipeline Test
+============================================================
+🔧 Initializing Pipeline Components...
+--------------------------------------------------
+1. Initializing Llama3-Med42-70B Client...
+2025-07-31 06:38:22,609 - llm_clients - INFO - Medical LLM client initialized with model: m42-health/Llama3-Med42-70B
+2025-07-31 06:38:22,609 - llm_clients - WARNING - Medical LLM Model: Research tool only. Not for professional medical diagnosis.
+   ✅ LLM client initialized successfully
+2. Initializing Retrieval System...
+2025-07-31 06:38:22,609 - retrieval - INFO - Initializing retrieval system...
+2025-07-31 06:38:22,621 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-07-31 06:38:22,621 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: NeuML/pubmedbert-base-embeddings
+2025-07-31 06:38:26,965 - retrieval - INFO - Embedding model loaded successfully
+2025-07-31 06:38:28,444 - retrieval - INFO - Chunks loaded successfully
+2025-07-31 06:38:28,532 - retrieval - INFO - Embeddings loaded successfully
+2025-07-31 06:38:28,533 - retrieval - INFO - Loaded existing emergency index
+2025-07-31 06:38:28,534 - retrieval - INFO - Loaded existing treatment index
+2025-07-31 06:38:28,534 - retrieval - INFO - Retrieval system initialized successfully
+   ✅ Retrieval system initialized successfully
+3. Initializing User Prompt Processor...
+2025-07-31 06:38:28,534 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-07-31 06:38:28,534 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: NeuML/pubmedbert-base-embeddings
+2025-07-31 06:38:30,716 - user_prompt - INFO - UserPromptProcessor initialized
+   ✅ User prompt processor initialized successfully
+
+🎉 All components initialized successfully!
+
+🚀 Starting Comprehensive Pipeline Test
+Total test cases: 6
+Test started at: 2025-07-31 06:38:22
+================================================================================
+
+🔍 test_001: Classic acute myocardial infarction query
+Query: 'how to treat acute MI?'
+------------------------------------------------------------
+Step 1: Extracting medical condition and keywords...
+2025-07-31 06:38:30,716 - llm_clients - INFO - Calling Medical LLM with query: how to treat acute MI?
+2025-07-31 06:39:12,449 - llm_clients - INFO - Raw LLM Response: The most representative condition: Acute Myocardial Infarction (AMI, or Heart Attack)
+
+For treatment guidance: Acute myocardial infarction is managed by cardiologists and emergency medical teams, not medical assistants. However, for informational purposes, primary treatments include:
+1. Reperfusion therapy: This may involve fibrinolysis (clot-busting medications) or percutaneous coronary intervention (PCI, such as angioplasty and stenting).
+2. Antiplatelet therapy
+2025-07-31 06:39:12,450 - llm_clients - INFO - Query Latency: 41.7327 seconds
+2025-07-31 06:39:12,450 - llm_clients - INFO - Extracted Condition: acute myocardial infarction
+   Condition: acute myocardial infarction
+   Emergency keywords: MI|chest pain|cardiac arrest
+   Treatment keywords: aspirin|nitroglycerin|thrombolytic|PCI
+   Source: predefined_mapping
+   Duration: 41.734s
+
+Step 2: User confirmation process...
+   Confirmation type: confirmation_needed
+
+Step 3: Executing retrieval...
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  1.46it/s]
+2025-07-31 06:39:13,227 - retrieval - INFO - Search results: Emergency=5, Treatment=5
+2025-07-31 06:39:13,228 - retrieval - INFO - Deduplication: Processing 10 results using text matching
+2025-07-31 06:39:13,228 - retrieval - INFO - Deduplication summary: 10 → 9 results (removed 1)
+   Search query: 'MI|chest pain|cardiac arrest aspirin|nitroglycerin|thrombolytic|PCI'
+   Total results: 9
+   Emergency results: 4
+   Treatment results: 5
+   Duration: 0.778s
+
+   Top 3 results:
+      1. Type: treatment, Distance: 0.6740
+         Text preview: ong term management abbreviations : ace : angiotensin converting enzyme ; arb : angiotensin receptor...
+      2. Type: treatment, Distance: 0.6792
+         Text preview: on ; pci : percutaneous coronary intervention ; po : per os ; stemi : st elevation myocardial infarc...
+      3. Type: treatment, Distance: 0.6904
+         Text preview: receptor blocker ; mi : myocardial infarction # do ' s - a pre - hospital ecg is recommended. if ste...
+
+✅ Test test_001 completed successfully (42.511s)
+
+🔍 test_002: Symptoms-based query requiring LLM analysis
+Query: 'patient with severe chest pain and shortness of breath'
+------------------------------------------------------------
+Step 1: Extracting medical condition and keywords...
+2025-07-31 06:39:13,228 - llm_clients - INFO - Calling Medical LLM with query: patient with severe chest pain and shortness of breath
+2025-07-31 06:39:31,525 - llm_clients - INFO - Raw LLM Response: Acute Coronary Syndrome (specifically, possible ST-Elevation Myocardial Infarction - STEMI, given severe chest pain, or non-STEMI/NST-Elevation Acute Coronary Syndrome if ST segments not elevated, based on ECG; shortness of breath indicates potential cardiac ischemia complication or concurrent pulmonary issue like cardiogenic pulmonary edema)
+
+Note: This response is for informational purposes only and should not replace immediate medical evaluation and diagnosis by a licensed physician. The patient needs
+2025-07-31 06:39:31,525 - llm_clients - INFO - Query Latency: 18.2971 seconds
+2025-07-31 06:39:31,525 - llm_clients - INFO - Extracted Condition: Acute Coronary Syndrome (specifically, possible ST-Elevation Myocardial Infarction - STEMI, given severe chest pain, or non-STEMI/NST-Elevation Acute Coronary Syndrome if ST segments not elevated, based on ECG; shortness of breath indicates potential cardiac ischemia complication or concurrent pulmonary issue like cardiogenic pulmonary edema)
+2025-07-31 06:39:31,525 - user_prompt - INFO - Starting semantic search fallback for query: 'patient with severe chest pain and shortness of breath'
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  7.70it/s]
+2025-07-31 06:39:32,392 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 06:39:32,402 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 13.86it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 17.53it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 17.22it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 57.51it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 17.23it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 58.05it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 59.09it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 60.88it/s]
+2025-07-31 06:39:32,729 - user_prompt - INFO - Inferred condition: None
+2025-07-31 06:39:32,729 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 06:39:32,729 - user_prompt - INFO - No suitable condition found in semantic search
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 16.77it/s]
+2025-07-31 06:39:33,251 - retrieval - INFO - Sliding window search: Found 5 results
+   Condition: generic medical query
+   Emergency keywords: medical|emergency
+   Treatment keywords: treatment|management
+   Source: generic_search
+   Duration: 20.033s
+
+Step 2: User confirmation process...
+   Confirmation type: confirmation_needed
+
+Step 3: Executing retrieval...
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 16.28it/s]
+2025-07-31 06:39:33,404 - retrieval - INFO - Search results: Emergency=5, Treatment=5
+2025-07-31 06:39:33,404 - retrieval - INFO - Deduplication: Processing 10 results using text matching
+2025-07-31 06:39:33,404 - retrieval - INFO - Deduplication summary: 10 → 9 results (removed 1)
+   Search query: 'medical|emergency treatment|management'
+   Total results: 9
+   Emergency results: 5
+   Treatment results: 4
+   Duration: 0.143s
+
+   Top 3 results:
+      1. Type: treatment, Distance: 0.7708
+         Text preview: and nurse practitioners who may or may not be formally trained in emergency medicine. they offer pri...
+      2. Type: emergency, Distance: 0.8056
+         Text preview: organization of emergency medical assistance emergency medical assistance is the first aid that is g...
+      3. Type: emergency, Distance: 0.8321
+         Text preview: ion to the emergency room ; - urgent situation that requires advanced medical care before transporta...
+
+✅ Test test_002 completed successfully (20.176s)
+
+🔍 test_003: Neurological emergency query
+Query: 'sudden neurological symptoms suggesting stroke'
+------------------------------------------------------------
+Step 1: Extracting medical condition and keywords...
+2025-07-31 06:39:33,404 - llm_clients - INFO - Calling Medical LLM with query: sudden neurological symptoms suggesting stroke
+2025-07-31 06:39:49,400 - llm_clients - INFO - Raw LLM Response: Cerebrovascular Accident (CVA), or Acute Ischemic Stroke
+
+(As a medical assistant, I'm limited to providing condition labels, not advice. In this case, the description given—sudden neurological symptoms suggestive of stroke—points to an acute ischemic stroke, also known as cerebrovascular accident (CVA). This diagnosis implies a blockage of blood flow to the brain, resulting in sudden neurological deficits.)
+
+**Please consult a qualified healthcare professional for evaluation and management.
+2025-07-31 06:39:49,403 - llm_clients - INFO - Query Latency: 15.9960 seconds
+2025-07-31 06:39:49,404 - llm_clients - INFO - Extracted Condition: Cerebrovascular Accident (CVA), or Acute Ischemic Stroke
+2025-07-31 06:39:49,405 - user_prompt - INFO - Starting semantic search fallback for query: 'sudden neurological symptoms suggesting stroke'
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  8.53it/s]
+2025-07-31 06:39:50,205 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 06:39:50,214 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 13.55it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 55.19it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 55.05it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.50it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.67it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.14it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 59.27it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 59.62it/s]
+2025-07-31 06:39:50,417 - user_prompt - INFO - Inferred condition: None
+2025-07-31 06:39:50,418 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 06:39:50,418 - user_prompt - INFO - No suitable condition found in semantic search
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 15.16it/s]
+2025-07-31 06:39:50,938 - retrieval - INFO - Sliding window search: Found 5 results
+   Condition: generic medical query
+   Emergency keywords: medical|emergency
+   Treatment keywords: treatment|management
+   Source: generic_search
+   Duration: 17.544s
+
+Step 2: User confirmation process...
+   Confirmation type: confirmation_needed
+
+Step 3: Executing retrieval...
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 46.02it/s]
+2025-07-31 06:39:50,972 - retrieval - INFO - Search results: Emergency=5, Treatment=5
+2025-07-31 06:39:50,972 - retrieval - INFO - Deduplication: Processing 10 results using text matching
+2025-07-31 06:39:50,972 - retrieval - INFO - Deduplication summary: 10 → 9 results (removed 1)
+   Search query: 'medical|emergency treatment|management'
+   Total results: 9
+   Emergency results: 5
+   Treatment results: 4
+   Duration: 0.025s
+
+   Top 3 results:
+      1. Type: treatment, Distance: 0.7708
+         Text preview: and nurse practitioners who may or may not be formally trained in emergency medicine. they offer pri...
+      2. Type: emergency, Distance: 0.8056
+         Text preview: organization of emergency medical assistance emergency medical assistance is the first aid that is g...
+      3. Type: emergency, Distance: 0.8321
+         Text preview: ion to the emergency room ; - urgent situation that requires advanced medical care before transporta...
+
+✅ Test test_003 completed successfully (17.569s)
+
+🔍 test_004: Protocol-specific stroke query
+Query: 'acute stroke management protocol'
+------------------------------------------------------------
+Step 1: Extracting medical condition and keywords...
+2025-07-31 06:39:50,973 - user_prompt - INFO - Matched predefined condition: acute stroke
+   Condition: acute stroke
+   Emergency keywords: stroke|neurological deficit|sudden weakness
+   Treatment keywords: tPA|thrombolysis|stroke unit care
+   Source: predefined_mapping
+   Duration: 0.000s
+
+Step 2: User confirmation process...
+   Confirmation type: confirmation_needed
+
+Step 3: Executing retrieval...
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 15.92it/s]
+2025-07-31 06:39:51,110 - retrieval - INFO - Search results: Emergency=5, Treatment=5
+2025-07-31 06:39:51,110 - retrieval - INFO - Deduplication: Processing 10 results using text matching
+2025-07-31 06:39:51,110 - retrieval - INFO - Deduplication summary: 10 → 9 results (removed 1)
+   Search query: 'stroke|neurological deficit|sudden weakness tPA|thrombolysis|stroke unit care'
+   Total results: 9
+   Emergency results: 5
+   Treatment results: 4
+   Duration: 0.137s
+
+   Top 3 results:
+      1. Type: treatment, Distance: 0.7389
+         Text preview: hree hours of the onset of stroke. early treatment ( within 90 minutes ) may be more likely to resul...
+      2. Type: treatment, Distance: 0.7401
+         Text preview: hree hours of the onset of stroke. early treatment ( within 90 minutes ) may be more likely to resul...
+      3. Type: emergency, Distance: 0.7685
+         Text preview: mproved outcomes for a broad spectrum of carefully selected clients who can be treated within three ...
+
+✅ Test test_004 completed successfully (0.137s)
+
+🔍 test_005: General symptom requiring LLM analysis
+Query: 'patient presenting with acute abdominal pain'
+------------------------------------------------------------
+Step 1: Extracting medical condition and keywords...
+2025-07-31 06:39:51,110 - llm_clients - INFO - Calling Medical LLM with query: patient presenting with acute abdominal pain
+2025-07-31 06:40:00,096 - llm_clients - INFO - Raw LLM Response: Acute Appendicitis
+
+(As a medical assistant, I identify the most representative condition here as acute appendicitis, given the patient's symptom of acute abdominal pain, particularly if localized in the right lower quadrant and accompanied by other typical signs like nausea, vomiting, fever, or guarding. However, this is not a definitive diagnosis and should be confirmed by a physician through clinical evaluation, imaging, or surgical findings.)
+2025-07-31 06:40:00,096 - llm_clients - INFO - Query Latency: 8.9862 seconds
+2025-07-31 06:40:00,097 - llm_clients - INFO - Extracted Condition: Acute Appendicitis
+2025-07-31 06:40:00,097 - user_prompt - INFO - Starting semantic search fallback for query: 'patient presenting with acute abdominal pain'
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 10.49it/s]
+2025-07-31 06:40:00,664 - retrieval - INFO - Sliding window search: Found 5 results
+2025-07-31 06:40:00,673 - user_prompt - INFO - Semantic search returned 5 results
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 15.57it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 50.55it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 47.08it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 62.74it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 45.91it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 61.25it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 58.38it/s]
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 64.09it/s]
+2025-07-31 06:40:00,876 - user_prompt - INFO - Inferred condition: None
+2025-07-31 06:40:00,876 - user_prompt - WARNING - Condition validation failed for: None
+2025-07-31 06:40:00,876 - user_prompt - INFO - No suitable condition found in semantic search
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 16.32it/s]
+2025-07-31 06:40:01,399 - retrieval - INFO - Sliding window search: Found 5 results
+   Condition: generic medical query
+   Emergency keywords: medical|emergency
+   Treatment keywords: treatment|management
+   Source: generic_search
+   Duration: 10.298s
+
+Step 2: User confirmation process...
+   Confirmation type: confirmation_needed
+
+Step 3: Executing retrieval...
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 45.41it/s]
+2025-07-31 06:40:01,432 - retrieval - INFO - Search results: Emergency=5, Treatment=5
+2025-07-31 06:40:01,432 - retrieval - INFO - Deduplication: Processing 10 results using text matching
+2025-07-31 06:40:01,432 - retrieval - INFO - Deduplication summary: 10 → 9 results (removed 1)
+   Search query: 'medical|emergency treatment|management'
+   Total results: 9
+   Emergency results: 5
+   Treatment results: 4
+   Duration: 0.025s
+
+   Top 3 results:
+      1. Type: treatment, Distance: 0.7708
+         Text preview: and nurse practitioners who may or may not be formally trained in emergency medicine. they offer pri...
+      2. Type: emergency, Distance: 0.8056
+         Text preview: organization of emergency medical assistance emergency medical assistance is the first aid that is g...
+      3. Type: emergency, Distance: 0.8321
+         Text preview: ion to the emergency room ; - urgent situation that requires advanced medical care before transporta...
+
+✅ Test test_005 completed successfully (10.322s)
+
+🔍 test_006: Specific condition with treatment focus
+Query: 'pulmonary embolism treatment guidelines'
+------------------------------------------------------------
+Step 1: Extracting medical condition and keywords...
+2025-07-31 06:40:01,432 - user_prompt - INFO - Matched predefined condition: pulmonary embolism
+   Condition: pulmonary embolism
+   Emergency keywords: chest pain|shortness of breath|sudden dyspnea
+   Treatment keywords: anticoagulation|heparin|embolectomy
+   Source: predefined_mapping
+   Duration: 0.000s
+
+Step 2: User confirmation process...
+   Confirmation type: confirmation_needed
+
+Step 3: Executing retrieval...
+Batches: 100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 16.27it/s]
+2025-07-31 06:40:01,562 - retrieval - INFO - Search results: Emergency=5, Treatment=5
+2025-07-31 06:40:01,562 - retrieval - INFO - Deduplication: Processing 10 results using text matching
+2025-07-31 06:40:01,562 - retrieval - INFO - Deduplication summary: 10 → 8 results (removed 2)
+   Search query: 'chest pain|shortness of breath|sudden dyspnea anticoagulation|heparin|embolectomy'
+   Total results: 8
+   Emergency results: 5
+   Treatment results: 3
+   Duration: 0.130s
+
+   Top 3 results:
+      1. Type: emergency, Distance: 0.8949
+         Text preview: algesics ( e. g. morphine, pethidine ) facilities for defibrillation ( df ) aspirin / anticoagulant ...
+      2. Type: treatment, Distance: 0.9196
+         Text preview: y proximal deep vein thrombosis leading to acute pulmonary embolism # # common causes of peripheral ...
+      3. Type: emergency, Distance: 0.9216
+         Text preview: ed or discolored skin in the affected leg - visible surface veins dvt usually involves the deep vein...
+
+✅ Test test_006 completed successfully (0.130s)
+
+================================================================================
+📊 COMPREHENSIVE TEST REPORT
+================================================================================
+🕐 Execution Summary:
+   Start time: 2025-07-31 06:38:22
+   End time: 2025-07-31 06:40:01
+   Total duration: 98.954s
+   Average per test: 16.492s
+
+📈 Test Results:
+   Total tests: 6
+   Successful: 6 ✅
+   Failed: 0 ❌
+   Success rate: 100.0%
+
+✅ Successful Tests Analysis:
+   Condition extraction sources:
+     - predefined_mapping: 3 tests
+     - generic_search: 3 tests
+   Performance metrics:
+     - Avg condition extraction: 14.935s
+     - Avg retrieval time: 0.206s
+
+   📋 test_001: Classic acute myocardial infarction query
+      Query: 'how to treat acute MI?'
+      Condition: acute myocardial infarction
+      Source: predefined_mapping
+      Results: 9 total (4 emergency, 5 treatment)
+      Duration: 42.511s
+
+   📋 test_002: Symptoms-based query requiring LLM analysis
+      Query: 'patient with severe chest pain and shortness of breath'
+      Condition: generic medical query
+      Source: generic_search
+      Results: 9 total (5 emergency, 4 treatment)
+      Duration: 20.176s
+
+   📋 test_003: Neurological emergency query
+      Query: 'sudden neurological symptoms suggesting stroke'
+      Condition: generic medical query
+      Source: generic_search
+      Results: 9 total (5 emergency, 4 treatment)
+      Duration: 17.569s
+
+   📋 test_004: Protocol-specific stroke query
+      Query: 'acute stroke management protocol'
+      Condition: acute stroke
+      Source: predefined_mapping
+      Results: 9 total (5 emergency, 4 treatment)
+      Duration: 0.137s
+
+   📋 test_005: General symptom requiring LLM analysis
+      Query: 'patient presenting with acute abdominal pain'
+      Condition: generic medical query
+      Source: generic_search
+      Results: 9 total (5 emergency, 4 treatment)
+      Duration: 10.322s
+
+   📋 test_006: Specific condition with treatment focus
+      Query: 'pulmonary embolism treatment guidelines'
+      Condition: pulmonary embolism
+      Source: predefined_mapping
+      Results: 8 total (5 emergency, 3 treatment)
+      Duration: 0.130s

tests/test_chunk_quality_analysis.py

@@ -0,0 +1,333 @@
+"""
+Chunk Quality Analysis Tests
+
+This module analyzes chunk quality and identifies issues with chunk length differences
+between emergency and treatment data processing methods.
+
+Author: OnCall.ai Team
+Date: 2025-07-28
+"""
+
+import sys
+import json
+import numpy as np
+from pathlib import Path
+from typing import List, Dict, Tuple
+import logging
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(levelname)s:%(name)s:%(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Add src to python path
+current_dir = Path(__file__).parent.resolve()
+project_root = current_dir.parent
+sys.path.append(str(project_root / "src"))
+
+from data_processing import DataProcessor #type: ignore
+
+class TestChunkQualityAnalysis:
+    
+    def setup_class(self):
+        """Initialize test environment"""
+        print("\n=== Phase 1: Setting up Chunk Quality Analysis ===")
+        self.base_dir = Path(__file__).parent.parent.resolve()
+        self.models_dir = self.base_dir / "models"
+        self.embeddings_dir = self.models_dir / "embeddings"
+        
+        print(f"• Base directory: {self.base_dir}")
+        print(f"• Models directory: {self.models_dir}")
+        
+        # Initialize processor
+        self.processor = DataProcessor(base_dir=str(self.base_dir))
+        print("• DataProcessor initialized")
+
+    def test_chunk_length_analysis(self):
+        """Detailed analysis of chunk length distribution"""
+        print("\n=== Phase 2: Chunk Length Distribution Analysis ===")
+        
+        try:
+            # Load chunk data
+            print("• Loading chunk data...")
+            with open(self.embeddings_dir / "emergency_chunks.json", 'r') as f:
+                emergency_chunks = json.load(f)
+            with open(self.embeddings_dir / "treatment_chunks.json", 'r') as f:
+                treatment_chunks = json.load(f)
+            
+            # Analyze emergency chunks
+            em_lengths = [len(chunk['text']) for chunk in emergency_chunks]
+            em_token_counts = [chunk.get('token_count', 0) for chunk in emergency_chunks]
+            
+            print(f"\n📊 Emergency Chunks Analysis:")
+            print(f"• Total chunks: {len(em_lengths):,}")
+            print(f"• Min length: {min(em_lengths)} chars")
+            print(f"• Max length: {max(em_lengths)} chars")
+            print(f"• Average length: {sum(em_lengths)/len(em_lengths):.2f} chars")
+            print(f"• Median length: {sorted(em_lengths)[len(em_lengths)//2]} chars")
+            
+            if any(em_token_counts):
+                avg_tokens = sum(em_token_counts)/len(em_token_counts)
+                print(f"• Average tokens: {avg_tokens:.2f}")
+                print(f"• Chars per token ratio: {(sum(em_lengths)/len(em_lengths)) / avg_tokens:.2f}")
+            
+            # Analyze treatment chunks
+            tr_lengths = [len(chunk['text']) for chunk in treatment_chunks]
+            
+            print(f"\n📊 Treatment Chunks Analysis:")
+            print(f"• Total chunks: {len(tr_lengths):,}")
+            print(f"• Min length: {min(tr_lengths)} chars")
+            print(f"• Max length: {max(tr_lengths)} chars")
+            print(f"• Average length: {sum(tr_lengths)/len(tr_lengths):.2f} chars")
+            print(f"• Median length: {sorted(tr_lengths)[len(tr_lengths)//2]} chars")
+            
+            # Length distribution comparison
+            em_avg = sum(em_lengths)/len(em_lengths)
+            tr_avg = sum(tr_lengths)/len(tr_lengths)
+            ratio = em_avg / tr_avg
+            
+            print(f"\n🔍 Length Distribution Comparison:")
+            print(f"• Emergency average: {em_avg:.0f} chars")
+            print(f"• Treatment average: {tr_avg:.0f} chars")
+            print(f"• Ratio (Emergency/Treatment): {ratio:.1f}x")
+            
+            # Length distribution buckets
+            print(f"\n📈 Length Distribution Buckets:")
+            buckets = [0, 100, 250, 500, 1000, 2000, 5000]
+            
+            for i in range(len(buckets)-1):
+                em_count = sum(1 for l in em_lengths if buckets[i] <= l < buckets[i+1])
+                tr_count = sum(1 for l in tr_lengths if buckets[i] <= l < buckets[i+1])
+                print(f"• {buckets[i]}-{buckets[i+1]} chars: Emergency={em_count}, Treatment={tr_count}")
+            
+            # Flag potential issues
+            if ratio > 5.0:
+                print(f"\n⚠️  WARNING: Emergency chunks are {ratio:.1f}x longer than treatment chunks!")
+                print("   This suggests different chunking strategies are being used.")
+            
+            print("✅ Chunk length analysis completed")
+            
+        except Exception as e:
+            print(f"❌ Error in chunk length analysis: {str(e)}")
+            raise
+
+    def test_chunking_method_comparison(self):
+        """Compare the two chunking methods on the same data"""
+        print("\n=== Phase 3: Chunking Method Comparison ===")
+        
+        try:
+            # Load data
+            print("• Loading dataset for comparison...")
+            self.processor.load_filtered_data()
+            
+            # Test on multiple samples for better analysis
+            sample_size = 5
+            samples = self.processor.treatment_data.head(sample_size)
+            
+            method1_results = []  # keyword_centered_chunks
+            method2_results = []  # dual_keyword_chunks
+            
+            print(f"• Testing {sample_size} samples with both methods...")
+            
+            for idx, row in samples.iterrows():
+                if not row.get('clean_text') or not row.get('treatment_matched'):
+                    continue
+                    
+                text_length = len(row['clean_text'])
+                emergency_kw = row.get('matched', '')
+                treatment_kw = row['treatment_matched']
+                
+                # Method 1: keyword_centered_chunks (Emergency method)
+                chunks1 = self.processor.create_keyword_centered_chunks(
+                    text=row['clean_text'],
+                    matched_keywords=emergency_kw,
+                    chunk_size=256,
+                    doc_id=f"test_{idx}"
+                )
+                
+                # Method 2: dual_keyword_chunks (Treatment method)
+                chunks2 = self.processor.create_dual_keyword_chunks(
+                    text=row['clean_text'],
+                    emergency_keywords=emergency_kw,
+                    treatment_keywords=treatment_kw,
+                    chunk_size=256,
+                    doc_id=f"test_{idx}"
+                )
+                
+                # Collect results
+                if chunks1:
+                    avg_len1 = sum(len(c['text']) for c in chunks1) / len(chunks1)
+                    method1_results.append({
+                        'doc_id': idx,
+                        'chunks_count': len(chunks1),
+                        'avg_length': avg_len1,
+                        'text_length': text_length
+                    })
+                
+                if chunks2:
+                    avg_len2 = sum(len(c['text']) for c in chunks2) / len(chunks2)
+                    method2_results.append({
+                        'doc_id': idx,
+                        'chunks_count': len(chunks2),
+                        'avg_length': avg_len2,
+                        'text_length': text_length
+                    })
+            
+            # Analysis results
+            print(f"\n📊 Method Comparison Results:")
+            
+            if method1_results:
+                avg_chunks1 = sum(r['chunks_count'] for r in method1_results) / len(method1_results)
+                avg_len1 = sum(r['avg_length'] for r in method1_results) / len(method1_results)
+                print(f"\n🔹 Keyword-Centered Method (Emergency):")
+                print(f"• Average chunks per document: {avg_chunks1:.1f}")
+                print(f"• Average chunk length: {avg_len1:.0f} chars")
+            
+            if method2_results:
+                avg_chunks2 = sum(r['chunks_count'] for r in method2_results) / len(method2_results)
+                avg_len2 = sum(r['avg_length'] for r in method2_results) / len(method2_results)
+                print(f"\n🔹 Dual-Keyword Method (Treatment):")
+                print(f"• Average chunks per document: {avg_chunks2:.1f}")
+                print(f"• Average chunk length: {avg_len2:.0f} chars")
+                
+                if method1_results:
+                    ratio = avg_len1 / avg_len2
+                    print(f"\n🔍 Length Ratio: {ratio:.1f}x (Method1 / Method2)")
+            
+            print("✅ Chunking method comparison completed")
+            
+        except Exception as e:
+            print(f"❌ Error in method comparison: {str(e)}")
+            raise
+
+    def test_token_vs_character_analysis(self):
+        """Analyze token vs character differences in chunking"""
+        print("\n=== Phase 4: Token vs Character Analysis ===")
+        
+        try:
+            # Load model for tokenization
+            print("• Loading embedding model for tokenization...")
+            self.processor.load_embedding_model()
+            
+            # Test sample texts
+            test_texts = [
+                "Patient presents with acute chest pain and shortness of breath.",
+                "Emergency treatment for myocardial infarction includes immediate medication.",
+                "The patient's vital signs show tachycardia and hypotension requiring intervention."
+            ]
+            
+            print(f"\n📊 Token vs Character Analysis:")
+            
+            total_chars = 0
+            total_tokens = 0
+            
+            for i, text in enumerate(test_texts, 1):
+                char_count = len(text)
+                token_count = len(self.processor.tokenizer.tokenize(text))
+                ratio = char_count / token_count if token_count > 0 else 0
+                
+                print(f"\nSample {i}:")
+                print(f"• Text: {text[:50]}...")
+                print(f"• Characters: {char_count}")
+                print(f"• Tokens: {token_count}")
+                print(f"• Chars/Token ratio: {ratio:.2f}")
+                
+                total_chars += char_count
+                total_tokens += token_count
+            
+            overall_ratio = total_chars / total_tokens
+            print(f"\n🔍 Overall Character/Token Ratio: {overall_ratio:.2f}")
+            
+            # Estimate chunk sizes
+            target_tokens = 256
+            estimated_chars = target_tokens * overall_ratio
+            
+            print(f"\n📏 Chunk Size Estimates:")
+            print(f"• Target tokens: {target_tokens}")
+            print(f"• Estimated characters: {estimated_chars:.0f}")
+            print(f"• Current emergency avg: 1842 chars ({1842/overall_ratio:.0f} estimated tokens)")
+            print(f"• Current treatment avg: 250 chars ({250/overall_ratio:.0f} estimated tokens)")
+            
+            # Recommendations
+            print(f"\n💡 Recommendations:")
+            if 1842/overall_ratio > 512:
+                print("⚠️  Emergency chunks may exceed model's 512 token limit!")
+            if 250/overall_ratio < 64:
+                print("⚠️  Treatment chunks may be too short for meaningful context!")
+            
+            print("✅ Token vs character analysis completed")
+            
+        except Exception as e:
+            print(f"❌ Error in token analysis: {str(e)}")
+            raise
+
+    def test_generate_recommendations(self):
+        """Generate recommendations based on analysis"""
+        print("\n=== Phase 5: Generating Recommendations ===")
+        
+        recommendations = []
+        
+        # Based on the known chunk length difference
+        recommendations.append({
+            'issue': 'Inconsistent chunk lengths',
+            'description': 'Emergency chunks (1842 chars) are 7x longer than treatment chunks (250 chars)',
+            'recommendation': 'Standardize both methods to use token-based chunking with consistent parameters',
+            'priority': 'HIGH'
+        })
+        
+        recommendations.append({
+            'issue': 'Different chunking strategies',
+            'description': 'Emergency uses keyword-centered (token-based), Treatment uses dual-keyword (character-based)',
+            'recommendation': 'Update dual_keyword_chunks to use tokenizer for consistent token-based chunking',
+            'priority': 'HIGH'
+        })
+        
+        recommendations.append({
+            'issue': 'Potential token limit overflow',
+            'description': 'Large chunks may exceed PubMedBERT 512 token limit',
+            'recommendation': 'Implement strict token-based chunking with overlap to prevent overflow',
+            'priority': 'MEDIUM'
+        })
+        
+        print(f"\n📋 Analysis Recommendations:")
+        for i, rec in enumerate(recommendations, 1):
+            print(f"\n{i}. {rec['issue']} [{rec['priority']}]")
+            print(f"   Problem: {rec['description']}")
+            print(f"   Solution: {rec['recommendation']}")
+        
+        print("\n✅ Recommendations generated")
+        return recommendations
+
+def main():
+    """Run all chunk quality analysis tests"""
+    print("\n" + "="*60)
+    print("CHUNK QUALITY ANALYSIS TEST SUITE")
+    print("="*60)
+    
+    test = TestChunkQualityAnalysis()
+    test.setup_class()
+    
+    try:
+        test.test_chunk_length_analysis()
+        test.test_chunking_method_comparison()
+        test.test_token_vs_character_analysis()
+        recommendations = test.test_generate_recommendations()
+        
+        print("\n" + "="*60)
+        print("🎉 ALL CHUNK QUALITY TESTS COMPLETED SUCCESSFULLY!")
+        print("="*60)
+        print(f"\nKey Finding: Chunk length inconsistency detected!")
+        print(f"Emergency: ~1842 chars, Treatment: ~250 chars (7x difference)")
+        print(f"Recommendation: Standardize to token-based chunking")
+        print("="*60)
+        
+    except Exception as e:
+        print("\n" + "="*60)
+        print("❌ CHUNK QUALITY TESTS FAILED!")
+        print(f"Error: {str(e)}")
+        print("="*60)
+
+if __name__ == "__main__":
+    main() 

tests/test_data_processing.py

@@ -12,7 +12,7 @@ import pandas as pd
 # Add src to path
 sys.path.append(str(Path(__file__).parent.parent.resolve() / "src"))
 
-from data_processing import DataProcessor
+from data_processing import DataProcessor #type: ignore
 import logging
 
 # Setup logging
@@ -80,7 +80,7 @@ def test_chunking():
                 chunks = processor.create_keyword_centered_chunks(
                     text=row['clean_text'],
                     matched_keywords=row['matched'],
-                    chunk_size=512,
+                    chunk_size=256,  # Updated to use 256 tokens
                     doc_id=str(row.get('id', idx))
                 )
                 emergency_chunks.extend(chunks)
@@ -97,7 +97,7 @@ def test_chunking():
                     text=row['clean_text'],
                     emergency_keywords=row.get('matched', ''),
                     treatment_keywords=row['treatment_matched'],
-                    chunk_size=512,
+                    chunk_size=256,  # Updated to use 256 tokens
                     doc_id=str(row.get('id', idx))
                 )
                 treatment_chunks.extend(chunks)
@@ -116,7 +116,7 @@ def test_chunking():
             sample_chunk = treatment_chunks[0]
             print(f"\nSample treatment chunk:")
             print(f"  Primary keyword: {sample_chunk['primary_keyword']}")
-            print(f"  Emergency keywords: {sample_chunk['emergency_keywords']}")
+            print(f"  Emergency keywords: {sample_chunk.get('emergency_keywords', '')}")
             print(f"  Text length: {len(sample_chunk['text'])}")
             print(f"  Text preview: {sample_chunk['text'][:100]}...")
         
@@ -186,18 +186,109 @@ def test_token_chunking():
         print(f"❌ Token chunking test failed: {e}")
         return False
 
+def test_dual_keyword_chunks():
+    """Test the enhanced dual keyword chunking functionality with token-based approach"""
+    print("\n" + "="*50)
+    print("TESTING DUAL KEYWORD CHUNKING")
+    print("="*50)
+    
+    try:
+        processor = DataProcessor()
+        processor.load_embedding_model()  # Need tokenizer for token count verification
+        
+        # Test case 1: Both emergency and treatment keywords
+        print("\nTest Case 1: Both Keywords")
+        text = "Patient with acute MI requires immediate IV treatment. Additional chest pain symptoms require aspirin administration."
+        emergency_kws = "MI|chest pain"
+        treatment_kws = "IV|aspirin"
+        
+        chunks = processor.create_dual_keyword_chunks(
+            text=text,
+            emergency_keywords=emergency_kws,
+            treatment_keywords=treatment_kws,
+            chunk_size=256
+        )
+        
+        # Verify chunk properties
+        for i, chunk in enumerate(chunks):
+            print(f"\nChunk {i+1}:")
+            # Verify source type
+            source_type = chunk.get('source_type')
+            assert source_type in ['emergency', 'treatment'], f"Invalid source_type: {source_type}"
+            print(f"• Source type: {source_type}")
+            
+            # Verify metadata for treatment chunks
+            if source_type == 'treatment':
+                contains_em = chunk.get('contains_emergency_kws', [])
+                contains_tr = chunk.get('contains_treatment_kws', [])
+                match_type = chunk.get('match_type')
+                print(f"• Contains Emergency: {contains_em}")
+                print(f"• Contains Treatment: {contains_tr}")
+                print(f"• Match Type: {match_type}")
+                assert match_type in ['both', 'emergency_only', 'treatment_only', 'none'], \
+                    f"Invalid match_type: {match_type}"
+            
+            # Verify token count
+            tokens = processor.tokenizer.tokenize(chunk['text'])
+            token_count = len(tokens)
+            print(f"• Token count: {token_count}")
+            # Allow for overlap
+            assert token_count <= 384, f"Chunk too large: {token_count} tokens"
+            
+            # Print text preview
+            print(f"• Text preview: {chunk['text'][:100]}...")
+        
+        # Test case 2: Emergency keywords only
+        print("\nTest Case 2: Emergency Only")
+        text = "Patient presents with severe chest pain and dyspnea."
+        emergency_kws = "chest pain"
+        treatment_kws = ""
+        
+        chunks = processor.create_dual_keyword_chunks(
+            text=text,
+            emergency_keywords=emergency_kws,
+            treatment_keywords=treatment_kws,
+            chunk_size=256
+        )
+        
+        assert len(chunks) > 0, "No chunks generated for emergency-only case"
+        print(f"✓ Generated {len(chunks)} chunks")
+        
+        # Test case 3: Treatment keywords only
+        print("\nTest Case 3: Treatment Only")
+        text = "Administer IV fluids and monitor response."
+        emergency_kws = ""
+        treatment_kws = "IV"
+        
+        chunks = processor.create_dual_keyword_chunks(
+            text=text,
+            emergency_keywords=emergency_kws,
+            treatment_keywords=treatment_kws,
+            chunk_size=256
+        )
+        
+        assert len(chunks) > 0, "No chunks generated for treatment-only case"
+        print(f"✓ Generated {len(chunks)} chunks")
+        
+        print("\n✅ All dual keyword chunking tests passed")
+        return True
+        
+    except Exception as e:
+        print(f"\n❌ Dual keyword chunking test failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
 def main():
     """Run all tests"""
     print("Starting data processing tests...\n")
     
-    # Import pandas here since it's used in chunking test
-    import pandas as pd
-    
     tests = [
         test_data_loading,
         test_chunking,
         test_model_loading,
-        test_token_chunking  # Added new test
+        test_token_chunking,
+        test_dual_keyword_chunks  # Added new test
     ]
     
     results = []

tests/test_embedding_and_index.py

@@ -1,29 +1,101 @@
+"""
+Basic embedding and index validation tests
+"""
+# 2025-07-28
+import sys
+from pathlib import Path
+#
+
 import numpy as np
 from annoy import AnnoyIndex
 import pytest
-from data_processing import DataProcessor
-
-@pytest.fixture(scope="module")
-def processor():
-    return DataProcessor(base_dir=".")
-
-def test_embedding_dimensions(processor):
-    # load emergency embeddings
-    emb = np.load(processor.models_dir / "embeddings" / "emergency_embeddings.npy")
-    expected_dim = processor.embedding_dim
-    assert emb.ndim == 2, f"Expected 2D array, got {emb.ndim}D"
-    assert emb.shape[1] == expected_dim, (
-        f"Expected embedding dimension {expected_dim}, got {emb.shape[1]}"
-    )
-
-def test_annoy_search(processor):
-    # load embeddings
-    emb = np.load(processor.models_dir / "embeddings" / "emergency_embeddings.npy")
-    # load Annoy index
-    idx = AnnoyIndex(processor.embedding_dim, 'angular')
-    idx.load(str(processor.models_dir / "indices" / "annoy" / "emergency_index.ann"))
-    # perform a sample query
-    query_vec = emb[0]
-    ids, distances = idx.get_nns_by_vector(query_vec, 5, include_distances=True)
-    assert len(ids) == 5
-    assert all(0 <= d <= 2 for d in distances)
+
+print("\n=== Phase 1: Initializing Test Environment ===")
+# add src to python path
+current_dir = Path(__file__).parent.resolve()
+project_root = current_dir.parent
+sys.path.append(str(project_root / "src"))
+
+print(f"• Current directory: {current_dir}")
+print(f"• Project root: {project_root}")
+print(f"• Python path: {sys.path}")
+
+from data_processing import DataProcessor #type: ignore
+
+
+class TestEmbeddingAndIndex:
+    def setup_class(self):
+        """初始化測試類"""
+        print("\n=== Phase 2: Setting up TestEmbeddingAndIndex ===")
+        self.base_dir = Path(__file__).parent.parent.resolve()
+        print(f"• Base directory: {self.base_dir}")
+        self.processor = DataProcessor(base_dir=str(self.base_dir))
+        print("• DataProcessor initialized")
+
+    def test_embedding_dimensions(self):
+        print("\n=== Phase 3: Testing Embedding Dimensions ===")
+        print("• Loading emergency embeddings...")
+        # load emergency embeddings
+        emb = np.load(self.processor.models_dir / "embeddings" / "emergency_embeddings.npy")
+        expected_dim = self.processor.embedding_dim
+        
+        print(f"• Loaded embedding shape: {emb.shape}")
+        print(f"• Expected dimension: {expected_dim}")
+        
+        assert emb.ndim == 2, f"Expected 2D array, got {emb.ndim}D"
+        assert emb.shape[1] == expected_dim, (
+            f"Expected embedding dimension {expected_dim}, got {emb.shape[1]}"
+        )
+        print("✅ Embedding dimensions test passed")
+
+    def test_annoy_search(self):
+        print("\n=== Phase 4: Testing Annoy Search ===")
+        print("• Loading embeddings...")
+        # load embeddings
+        emb = np.load(self.processor.models_dir / "embeddings" / "emergency_embeddings.npy")
+        print(f"• Loaded embeddings shape: {emb.shape}")
+        
+        print("• Loading Annoy index...")
+        # load Annoy index
+        idx = AnnoyIndex(self.processor.embedding_dim, 'angular')
+        index_path = self.processor.models_dir / "indices" / "annoy" / "emergency_index.ann"
+        print(f"• Index path: {index_path}")
+        idx.load(str(index_path))
+        
+        print("• Performing sample query...")
+        # perform a sample query
+        query_vec = emb[0]
+        ids, distances = idx.get_nns_by_vector(query_vec, 5, include_distances=True)
+        
+        print(f"• Search results:")
+        print(f"  - Found IDs: {ids}")
+        print(f"  - Distances: {[f'{d:.4f}' for d in distances]}")
+        
+        assert len(ids) == 5, f"Expected 5 results, got {len(ids)}"
+        assert all(0 <= d <= 2 for d in distances), "Invalid distance values"
+        print("✅ Annoy search test passed")
+
+def main():
+    """Run tests manually"""
+    print("\n" + "="*50)
+    print("Starting Embedding and Index Tests")
+    print("="*50)
+    
+    test = TestEmbeddingAndIndex()
+    test.setup_class()  # 手動初始化
+    
+    try:
+        test.test_embedding_dimensions()
+        test.test_annoy_search()
+        print("\n" + "="*50)
+        print("🎉 All tests completed successfully!")
+        print("="*50)
+        
+    except Exception as e:
+        print("\n" + "="*50)
+        print("❌ Tests failed!")
+        print(f"Error: {str(e)}")
+        print("="*50)
+
+if __name__ == "__main__":
+    main()

tests/test_embedding_validation.py

@@ -7,14 +7,27 @@ import numpy as np
 import json
 import logging
 import os
+import sys
 from pathlib import Path
 from typing import Tuple, List, Optional
 from annoy import AnnoyIndex
 from sentence_transformers import SentenceTransformer
 
+print("\n=== Phase 1: Initializing Test Environment ===")
+# Add src to python path
+current_dir = Path(__file__).parent.resolve()
+project_root = current_dir.parent
+sys.path.append(str(project_root / "src"))
+
+print(f"• Current directory: {current_dir}")
+print(f"• Project root: {project_root}")
+print(f"• Python path added: {project_root / 'src'}")
+
 class TestEmbeddingValidation:
     def setup_class(self):
         """Initialize test environment with necessary data and models."""
+        print("\n=== Phase 2: Setting up Test Environment ===")
+        
         # Setup logging
         logging.basicConfig(
             level=logging.DEBUG,
@@ -24,43 +37,57 @@ class TestEmbeddingValidation:
         self.logger = logging.getLogger(__name__)
         
         # Define base paths
-        self.project_root = Path(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+        self.project_root = Path(__file__).parent.parent.resolve()
         self.models_dir = self.project_root / "models"
         self.embeddings_dir = self.models_dir / "embeddings"
         self.indices_dir = self.models_dir / "indices" / "annoy"
 
+        print(f"• Project root: {self.project_root}")
+        print(f"• Models directory: {self.models_dir}")
+        print(f"• Embeddings directory: {self.embeddings_dir}")
+
         self.logger.info(f"Project root: {self.project_root}")
         self.logger.info(f"Models directory: {self.models_dir}")
         self.logger.info(f"Embeddings directory: {self.embeddings_dir}")
         
         try:
             # Check directory existence
+            print("• Checking directory existence...")
             if not self.embeddings_dir.exists():
                 raise FileNotFoundError(f"Embeddings directory not found at: {self.embeddings_dir}")
             if not self.indices_dir.exists():
                 raise FileNotFoundError(f"Indices directory not found at: {self.indices_dir}")
             
             # Load embeddings
+            print("• Loading embeddings...")
             self.emergency_emb = np.load(self.embeddings_dir / "emergency_embeddings.npy")
             self.treatment_emb = np.load(self.embeddings_dir / "treatment_embeddings.npy")
             
             # Load chunks
+            print("• Loading chunk metadata...")
             with open(self.embeddings_dir / "emergency_chunks.json", 'r') as f:
                 self.emergency_chunks = json.load(f)
             with open(self.embeddings_dir / "treatment_chunks.json", 'r') as f:
                 self.treatment_chunks = json.load(f)
                 
             # Initialize model
+            print("• Loading PubMedBERT model...")
             self.model = SentenceTransformer("NeuML/pubmedbert-base-embeddings")
             
+            print(f"• Emergency embeddings shape: {self.emergency_emb.shape}")
+            print(f"• Treatment embeddings shape: {self.treatment_emb.shape}")
+            print("✅ Test environment initialized successfully")
+            
             self.logger.info("Test environment initialized successfully")
             self.logger.info(f"Emergency embeddings shape: {self.emergency_emb.shape}")
             self.logger.info(f"Treatment embeddings shape: {self.treatment_emb.shape}")
             
         except FileNotFoundError as e:
+            print(f"❌ File not found: {e}")
             self.logger.error(f"File not found: {e}")
             raise
         except Exception as e:
+            print(f"❌ Error during initialization: {e}")
             self.logger.error(f"Error during initialization: {e}")
             raise
     
@@ -84,20 +111,28 @@ class TestEmbeddingValidation:
     
     def test_embedding_dimensions(self):
         """Test embedding dimensions and data quality."""
+        print("\n=== Phase 3: Embedding Validation ===")
         self.logger.info("\n=== Embedding Validation Report ===")
         
         try:
             # Basic dimension checks
+            print("• Checking embedding dimensions...")
             assert self.emergency_emb.shape[1] == 768, "Emergency embedding dimension should be 768"
             assert self.treatment_emb.shape[1] == 768, "Treatment embedding dimension should be 768"
+            print(f"✓ Emergency dimensions: {self.emergency_emb.shape}")
+            print(f"✓ Treatment dimensions: {self.treatment_emb.shape}")
             
             # Count verification
+            print("• Verifying chunk count consistency...")
             assert len(self.emergency_chunks) == self.emergency_emb.shape[0], \
                 "Emergency chunks count mismatch"
             assert len(self.treatment_chunks) == self.treatment_emb.shape[0], \
                 "Treatment chunks count mismatch"
+            print(f"✓ Emergency: {len(self.emergency_chunks)} chunks = {self.emergency_emb.shape[0]} embeddings")
+            print(f"✓ Treatment: {len(self.treatment_chunks)} chunks = {self.treatment_emb.shape[0]} embeddings")
             
             # Data quality checks
+            print("• Performing data quality checks...")
             for name, emb in [("Emergency", self.emergency_emb), 
                              ("Treatment", self.treatment_emb)]:
                 # Check for NaN and Inf
@@ -105,25 +140,35 @@ class TestEmbeddingValidation:
                 assert not np.isinf(emb).any(), f"{name} contains Inf values"
                 
                 # Value distribution analysis
+                print(f"\n📊 {name} Embeddings Statistics:")
+                print(f"• Range: {np.min(emb):.3f} to {np.max(emb):.3f}")
+                print(f"• Mean: {np.mean(emb):.3f}")
+                print(f"• Std: {np.std(emb):.3f}")
+                
                 self.logger.info(f"\n{name} Embeddings Statistics:")
                 self.logger.info(f"- Range: {np.min(emb):.3f} to {np.max(emb):.3f}")
                 self.logger.info(f"- Mean: {np.mean(emb):.3f}")
                 self.logger.info(f"- Std: {np.std(emb):.3f}")
             
+            print("\n✅ All embedding validations passed")
             self.logger.info("\n✅ All embedding validations passed")
             
         except AssertionError as e:
+            print(f"❌ Validation failed: {str(e)}")
             self.logger.error(f"Validation failed: {str(e)}")
             raise
     
     def test_multiple_known_item_search(self):
         """Test ANNOY search with multiple random samples."""
+        print("\n=== Phase 4: Multiple Known-Item Search Test ===")
         self.logger.info("\n=== Multiple Known-Item Search Test ===")
         
+        print("• Loading emergency index...")
         emergency_index = AnnoyIndex(768, 'angular')
         emergency_index.load(str(self.indices_dir / "emergency_index.ann"))
         
         # Test 20 random samples
+        print("• Selecting 20 random samples for self-retrieval test...")
         test_indices = np.random.choice(
             self.emergency_emb.shape[0], 
             size=20,
@@ -131,36 +176,45 @@ class TestEmbeddingValidation:
         )
         
         success_count = 0
-        for test_idx in test_indices:
+        print("• Testing self-retrieval for each sample...")
+        for i, test_idx in enumerate(test_indices, 1):
             try:
                 test_emb = self.emergency_emb[test_idx]
                 indices, distances = self._safe_search(emergency_index, test_emb)
                 
                 if indices is None:
+                    print(f"  {i}/20: ❌ Search failed for index {test_idx}")
                     continue
                 
                 # Verify self-retrieval
                 assert indices[0] == test_idx, f"Self-retrieval failed for index {test_idx}"
                 assert distances[0] < 0.0001, f"Self-distance too large for index {test_idx}"
                 success_count += 1
+                print(f"  {i}/20: ✓ Index {test_idx} (distance: {distances[0]:.6f})")
                 
             except AssertionError as e:
+                print(f"  {i}/20: ❌ Index {test_idx} failed: {str(e)}")
                 self.logger.warning(f"Test failed for index {test_idx}: {str(e)}")
         
+        print(f"\n📊 Self-Retrieval Results: {success_count}/20 tests passed ({success_count/20*100:.1f}%)")
         self.logger.info(f"\n✅ {success_count}/20 self-retrieval tests passed")
         assert success_count >= 18, "Less than 90% of self-retrieval tests passed"
+        print("✅ Multiple known-item search test passed")
     
     def test_balanced_cross_dataset_search(self):
         """Test search across both emergency and treatment datasets."""
+        print("\n=== Phase 5: Cross-Dataset Search Test ===")
         self.logger.info("\n=== Balanced Cross-Dataset Search Test ===")
         
         # Initialize indices
+        print("• Loading ANNOY indices...")
         emergency_index = AnnoyIndex(768, 'angular')
         treatment_index = AnnoyIndex(768, 'angular')
         
         try:
             emergency_index.load(str(self.indices_dir / "emergency_index.ann"))
             treatment_index.load(str(self.indices_dir / "treatment_index.ann"))
+            print("✓ Emergency and treatment indices loaded")
             
             # Test queries
             test_queries = [
@@ -169,45 +223,75 @@ class TestEmbeddingValidation:
                 "What are the emergency procedures for anaphylactic shock?"
             ]
             
-            for query in test_queries:
-                print(f"\n\n=== Query: {query} ===")
+            print(f"• Testing {len(test_queries)} medical queries...")
+            
+            for query_num, query in enumerate(test_queries, 1):
+                print(f"\n🔍 Query {query_num}/3: {query}")
                 
                 # Generate query vector
+                print("• Generating query embedding...")
                 query_emb = self.model.encode([query])[0]
                 
                 # Get top-5 results from each dataset
+                print("• Searching both datasets...")
                 e_indices, e_distances = self._safe_search(emergency_index, query_emb, k=5)
                 t_indices, t_distances = self._safe_search(treatment_index, query_emb, k=5)
                 
                 if None in [e_indices, e_distances, t_indices, t_distances]:
+                    print("❌ Search failed for one or both datasets")
                     self.logger.error("Search failed for one or both datasets")
                     continue
                 
                 # Print first sentence of each result
-                print("\nEmergency Dataset Results:")
+                print(f"\n📋 Emergency Dataset Results:")
                 for i, (idx, dist) in enumerate(zip(e_indices, e_distances), 1):
                     text = self.emergency_chunks[idx]['text']
                     first_sentence = text.split('.')[0] + '.'
-                    print(f"\nE-{i} (distance: {dist:.3f}):")
-                    print(first_sentence)
+                    print(f"  E-{i} (distance: {dist:.3f}): {first_sentence[:80]}...")
                 
-                print("\nTreatment Dataset Results:")
+                print(f"\n📋 Treatment Dataset Results:")
                 for i, (idx, dist) in enumerate(zip(t_indices, t_distances), 1):
                     text = self.treatment_chunks[idx]['text']
                     first_sentence = text.split('.')[0] + '.'
-                    print(f"\nT-{i} (distance: {dist:.3f}):")
-                    print(first_sentence)
+                    print(f"  T-{i} (distance: {dist:.3f}): {first_sentence[:80]}...")
+                
+                print("✓ Query completed")
             
         except Exception as e:
+            print(f"❌ Test failed: {str(e)}")
             self.logger.error(f"Test failed: {str(e)}")
             raise
         else:
+            print("\n✅ Cross-dataset search test completed")
             self.logger.info("\n✅ Cross-dataset search test completed")
 
-if __name__ == "__main__":
-    # Manual test execution
+def main():
+    """Run all embedding validation tests"""
+    print("\n" + "="*60)
+    print("COMPREHENSIVE EMBEDDING VALIDATION TEST SUITE")
+    print("="*60)
+    
     test = TestEmbeddingValidation()
     test.setup_class()
-    test.test_embedding_dimensions()
-    test.test_multiple_known_item_search()
-    test.test_balanced_cross_dataset_search() 
+    
+    try:
+        test.test_embedding_dimensions()
+        test.test_multiple_known_item_search()
+        test.test_balanced_cross_dataset_search() 
+        
+        print("\n" + "="*60)
+        print("🎉 ALL EMBEDDING VALIDATION TESTS COMPLETED SUCCESSFULLY!")
+        print("="*60)
+        print("✅ Embedding dimensions validated")
+        print("✅ Self-retrieval accuracy confirmed")
+        print("✅ Cross-dataset search functionality verified")
+        print("="*60)
+        
+    except Exception as e:
+        print("\n" + "="*60)
+        print("❌ EMBEDDING VALIDATION TESTS FAILED!")
+        print(f"Error: {str(e)}")
+        print("="*60)
+
+if __name__ == "__main__":
+    main() 

tests/test_end_to_end_pipeline.py

@@ -0,0 +1,473 @@
+#!/usr/bin/env python3
+"""
+End-to-End Pipeline Script Test for OnCall.ai
+
+Tests the complete pipeline:
+User Input → UserPrompt Processing → Retrieval → Generation → Structured Medical Advice
+
+This script validates the entire workflow with realistic medical queries,
+simulating the user confirmation process and generating final medical advice.
+
+Author: OnCall.ai Team
+Date: 2025-07-31
+"""
+
+import sys
+import os
+from pathlib import Path
+import logging
+import json
+import traceback
+from datetime import datetime
+from typing import Dict, List, Any, Optional
+
+# Add src directory to Python path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import all pipeline modules
+try:
+    from user_prompt import UserPromptProcessor
+    from retrieval import BasicRetrievalSystem
+    from llm_clients import llm_Med42_70BClient
+    from generation import MedicalAdviceGenerator
+    from medical_conditions import CONDITION_KEYWORD_MAPPING
+except ImportError as e:
+    print(f"❌ Import Error: {e}")
+    print(f"Current working directory: {os.getcwd()}")
+    print(f"Python path: {sys.path}")
+    sys.exit(1)
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(),
+        logging.FileHandler(project_root / 'tests' / 'end_to_end_pipeline.log')
+    ]
+)
+logger = logging.getLogger(__name__)
+
+class EndToEndPipelineTest:
+    """Complete pipeline test with realistic medical scenarios"""
+    
+    def __init__(self):
+        """Initialize test suite"""
+        self.start_time = datetime.now()
+        self.test_results = []
+        self.components_initialized = False
+        
+        # Pipeline components
+        self.llm_client = None
+        self.retrieval_system = None
+        self.user_prompt_processor = None
+        self.medical_generator = None
+        
+    def initialize_complete_pipeline(self):
+        """Initialize all pipeline components"""
+        print("🔧 Initializing Complete OnCall.ai Pipeline...")
+        print("-" * 60)
+        
+        try:
+            # Initialize LLM client
+            print("1. Initializing Med42-70B Client...")
+            self.llm_client = llm_Med42_70BClient()
+            print("   ✅ Med42-70B client ready")
+            
+            # Initialize retrieval system
+            print("2. Initializing Dual-Index Retrieval System...")
+            self.retrieval_system = BasicRetrievalSystem()
+            print("   ✅ Emergency & Treatment indices loaded")
+            
+            # Initialize user prompt processor
+            print("3. Initializing Multi-Level Prompt Processor...")
+            self.user_prompt_processor = UserPromptProcessor(
+                llm_client=self.llm_client,
+                retrieval_system=self.retrieval_system
+            )
+            print("   ✅ Fallback validation system ready")
+            
+            # Initialize medical advice generator
+            print("4. Initializing Medical Advice Generator...")
+            self.medical_generator = MedicalAdviceGenerator(
+                llm_client=self.llm_client
+            )
+            print("   ✅ RAG generation system ready")
+            
+            self.components_initialized = True
+            print(f"\n🎉 Complete pipeline initialized successfully!")
+            
+        except Exception as e:
+            logger.error(f"Pipeline initialization failed: {e}")
+            print(f"❌ Initialization failed: {e}")
+            traceback.print_exc()
+            self.components_initialized = False
+    
+    def get_realistic_test_queries(self) -> List[Dict[str, Any]]:
+        """Define realistic medical queries for end-to-end testing"""
+        return [
+            {
+                "id": "e2e_001",
+                "query": "How to treat acute myocardial infarction in emergency department?",
+                "description": "Classic cardiac emergency with treatment focus",
+                "expected_intention": "treatment",
+                "category": "cardiac_emergency",
+                "simulated_confirmation": "yes"
+            },
+            {
+                "id": "e2e_002", 
+                "query": "Patient presenting with severe chest pain and shortness of breath",
+                "description": "Symptom-based emergency requiring assessment and treatment",
+                "expected_intention": "diagnosis",
+                "category": "multi_symptom",
+                "simulated_confirmation": "yes"
+            },
+            {
+                "id": "e2e_003",
+                "query": "What are the emergency protocols for acute stroke management?",
+                "description": "Neurological emergency with protocol focus",
+                "expected_intention": "treatment", 
+                "category": "neurological_emergency",
+                "simulated_confirmation": "yes"
+            },
+            {
+                "id": "e2e_004",
+                "query": "Differential diagnosis for sudden onset chest pain in young adult",
+                "description": "Diagnostic reasoning query",
+                "expected_intention": "diagnosis",
+                "category": "differential_diagnosis",
+                "simulated_confirmation": "yes"
+            },
+            {
+                "id": "e2e_005",
+                "query": "Emergency management of pulmonary embolism",
+                "description": "Pulmonary emergency requiring immediate intervention",
+                "expected_intention": "treatment",
+                "category": "pulmonary_emergency", 
+                "simulated_confirmation": "yes"
+            },
+            {
+                "id": "e2e_006",
+                "query": "How to cook pasta properly?",
+                "description": "Non-medical query - should be rejected",
+                "expected_intention": None,
+                "category": "non_medical",
+                "simulated_confirmation": "reject_expected"
+            }
+        ]
+    
+    def run_scripted_end_to_end_tests(self):
+        """Execute complete end-to-end tests with realistic queries"""
+        if not self.components_initialized:
+            print("❌ Cannot run tests: pipeline not initialized")
+            return
+            
+        test_queries = self.get_realistic_test_queries()
+        
+        print(f"\n🚀 Starting End-to-End Pipeline Tests")
+        print(f"Total test scenarios: {len(test_queries)}")
+        print(f"Test started at: {self.start_time.strftime('%Y-%m-%d %H:%M:%S')}")
+        print("=" * 80)
+        
+        # Execute all tests
+        for test_case in test_queries:
+            result = self._execute_single_pipeline_test(test_case)
+            self.test_results.append(result)
+        
+        # Generate comprehensive report
+        self._generate_end_to_end_report()
+        self._save_end_to_end_results()
+    
+    def _execute_single_pipeline_test(self, test_case: Dict[str, Any]) -> Dict[str, Any]:
+        """Execute single test through complete pipeline"""
+        test_id = test_case["id"]
+        query = test_case["query"]
+        
+        print(f"\n🧪 {test_id}: {test_case['description']}")
+        print(f"Query: '{query}'")
+        print(f"Expected: {test_case['expected_intention']} intention")
+        print("-" * 70)
+        
+        pipeline_start = datetime.now()
+        result = {
+            "test_id": test_id,
+            "test_case": test_case,
+            "timestamp": datetime.now().isoformat(),
+            "success": False,
+            "error": None,
+            "total_pipeline_time": 0,
+            "pipeline_steps": {}
+        }
+        
+        try:
+            # STEP 1: User Prompt Processing
+            print("   🎯 Step 1: Condition extraction and validation...")
+            step1_start = datetime.now()
+            
+            condition_result = self.user_prompt_processor.extract_condition_keywords(query)
+            step1_time = (datetime.now() - step1_start).total_seconds()
+            
+            result["pipeline_steps"]["condition_extraction"] = {
+                "duration": step1_time,
+                "result": condition_result,
+                "condition_found": bool(condition_result.get('condition'))
+            }
+            
+            print(f"      Condition: {condition_result.get('condition', 'None')}")
+            print(f"      Keywords: Emergency='{condition_result.get('emergency_keywords', 'None')}', Treatment='{condition_result.get('treatment_keywords', 'None')}'")
+            print(f"      Time: {step1_time:.3f}s")
+            
+            # Check if this is a non-medical query that should be rejected
+            if condition_result.get('type') == 'invalid_query':
+                print("      🚫 Non-medical query correctly rejected")
+                result["pipeline_steps"]["rejection"] = {
+                    "reason": "non_medical_query",
+                    "message": condition_result.get('message', '')
+                }
+                result["success"] = test_case['category'] == 'non_medical'
+                return result
+            
+            # STEP 2: User Confirmation (Simulated)
+            print("   🤝 Step 2: User confirmation (simulated as 'yes')...")
+            confirmation = self.user_prompt_processor.handle_user_confirmation(condition_result)
+            
+            result["pipeline_steps"]["confirmation"] = {
+                "type": confirmation.get('type', 'unknown'),
+                "simulated_response": test_case['simulated_confirmation']
+            }
+            
+            if not condition_result.get('condition'):
+                print("      ⚠️  No condition extracted, skipping retrieval and generation")
+                result["pipeline_steps"]["pipeline_stopped"] = "no_condition"
+                return result
+            
+            # STEP 3: Retrieval
+            print("   🔍 Step 3: Medical guideline retrieval...")
+            step3_start = datetime.now()
+            
+            search_query = f"{condition_result.get('emergency_keywords', '')} {condition_result.get('treatment_keywords', '')}".strip()
+            if not search_query:
+                search_query = condition_result.get('condition', query)
+            
+            retrieval_results = self.retrieval_system.search(search_query, top_k=5)
+            step3_time = (datetime.now() - step3_start).total_seconds()
+            
+            processed_results = retrieval_results.get('processed_results', [])
+            emergency_count = len([r for r in processed_results if r.get('type') == 'emergency'])
+            treatment_count = len([r for r in processed_results if r.get('type') == 'treatment'])
+            
+            result["pipeline_steps"]["retrieval"] = {
+                "duration": step3_time,
+                "search_query": search_query,
+                "total_results": len(processed_results),
+                "emergency_results": emergency_count,
+                "treatment_results": treatment_count
+            }
+            
+            print(f"      Search Query: '{search_query}'")
+            print(f"      Results: {len(processed_results)} total ({emergency_count} emergency, {treatment_count} treatment)")
+            print(f"      Time: {step3_time:.3f}s")
+            
+            # STEP 4: Medical Advice Generation
+            print("   🧠 Step 4: Medical advice generation...")
+            step4_start = datetime.now()
+            
+            # Determine intention (simulate intelligent detection)
+            intention = test_case.get('expected_intention')
+            
+            medical_advice = self.medical_generator.generate_medical_advice(
+                user_query=query,
+                retrieval_results=retrieval_results,
+                intention=intention
+            )
+            step4_time = (datetime.now() - step4_start).total_seconds()
+            
+            result["pipeline_steps"]["generation"] = {
+                "duration": step4_time,
+                "intention_used": intention,
+                "confidence_score": medical_advice.get('confidence_score', 0.0),
+                "advice_length": len(medical_advice.get('medical_advice', '')),
+                "chunks_used": medical_advice.get('query_metadata', {}).get('total_chunks_used', 0)
+            }
+            
+            print(f"      Intention: {intention}")
+            print(f"      Confidence: {medical_advice.get('confidence_score', 0.0):.2f}")
+            print(f"      Advice Length: {len(medical_advice.get('medical_advice', ''))} chars")
+            print(f"      Chunks Used: {medical_advice.get('query_metadata', {}).get('total_chunks_used', 0)}")
+            print(f"      Time: {step4_time:.3f}s")
+            
+            # STEP 5: Results Summary
+            total_time = (datetime.now() - pipeline_start).total_seconds()
+            result["total_pipeline_time"] = total_time
+            result["final_medical_advice"] = medical_advice
+            result["success"] = True
+            
+            print(f"\n   ✅ Pipeline completed successfully!")
+            print(f"   📊 Total Time: {total_time:.3f}s")
+            print(f"   🩺 Medical Advice Preview:")
+            print(f"      {medical_advice.get('medical_advice', 'No advice generated')[:150]}...")
+            
+        except Exception as e:
+            total_time = (datetime.now() - pipeline_start).total_seconds()
+            result["total_pipeline_time"] = total_time
+            result["error"] = str(e)
+            result["traceback"] = traceback.format_exc()
+            
+            logger.error(f"Pipeline test {test_id} failed: {e}")
+            print(f"   ❌ Pipeline failed: {e}")
+            
+        return result
+    
+    def _determine_extraction_source(self, condition_result: Dict) -> str:
+        """Determine how the condition was extracted"""
+        if condition_result.get('semantic_confidence') is not None:
+            return "semantic_search"
+        elif condition_result.get('generic_confidence') is not None:
+            return "generic_search"
+        elif condition_result.get('condition') in CONDITION_KEYWORD_MAPPING:
+            return "predefined_mapping"
+        else:
+            return "llm_extraction"
+    
+    def _generate_end_to_end_report(self):
+        """Generate comprehensive end-to-end test report"""
+        end_time = datetime.now()
+        total_duration = (end_time - self.start_time).total_seconds()
+        
+        successful_tests = [r for r in self.test_results if r['success']]
+        failed_tests = [r for r in self.test_results if not r['success']]
+        
+        print("\n" + "=" * 80)
+        print("📊 END-TO-END PIPELINE TEST REPORT")
+        print("=" * 80)
+        
+        # Overall Statistics
+        print(f"🕐 Execution Summary:")
+        print(f"   Test session duration: {total_duration:.3f}s")
+        print(f"   Average per test: {total_duration/len(self.test_results):.3f}s")
+        
+        print(f"\n📈 Pipeline Results:")
+        print(f"   Total tests: {len(self.test_results)}")
+        print(f"   Successful: {len(successful_tests)} ✅")
+        print(f"   Failed: {len(failed_tests)} ❌")
+        print(f"   Success rate: {len(successful_tests)/len(self.test_results)*100:.1f}%")
+        
+        # Performance Analysis
+        if successful_tests:
+            print(f"\n⚡ Performance Analysis:")
+            
+            # Calculate average times for each step
+            step_times = {}
+            for result in successful_tests:
+                for step_name, step_data in result.get('pipeline_steps', {}).items():
+                    if 'duration' in step_data:
+                        if step_name not in step_times:
+                            step_times[step_name] = []
+                        step_times[step_name].append(step_data['duration'])
+            
+            for step_name, times in step_times.items():
+                avg_time = sum(times) / len(times)
+                print(f"   {step_name.replace('_', ' ').title()}: {avg_time:.3f}s average")
+            
+            # Overall pipeline performance
+            total_times = [r['total_pipeline_time'] for r in successful_tests]
+            avg_total = sum(total_times) / len(total_times)
+            print(f"   Complete Pipeline: {avg_total:.3f}s average")
+        
+        # Detailed Results
+        print(f"\n📝 Detailed Test Results:")
+        for result in self.test_results:
+            test_case = result['test_case']
+            status = "✅ PASS" if result['success'] else "❌ FAIL"
+            
+            print(f"\n   📋 {result['test_id']}: {status}")
+            print(f"      Query: '{test_case['query']}'")
+            print(f"      Category: {test_case['category']}")
+            print(f"      Total Time: {result['total_pipeline_time']:.3f}s")
+            
+            if result['success']:
+                steps = result.get('pipeline_steps', {})
+                if 'condition_extraction' in steps:
+                    condition = steps['condition_extraction']['result'].get('condition', 'None')
+                    print(f"      Condition Extracted: {condition}")
+                
+                if 'generation' in steps:
+                    confidence = steps['generation'].get('confidence_score', 0.0)
+                    chunks = steps['generation'].get('chunks_used', 0)
+                    print(f"      Generation: {confidence:.2f} confidence, {chunks} chunks")
+                
+                if 'final_medical_advice' in result:
+                    advice = result['final_medical_advice'].get('medical_advice', '')
+                    print(f"      Advice Preview: {advice[:100]}...")
+            else:
+                if result.get('error'):
+                    print(f"      Error: {result['error']}")
+                elif 'rejection' in result.get('pipeline_steps', {}):
+                    print(f"      Rejected: {result['pipeline_steps']['rejection']['reason']}")
+        
+        print("\n" + "=" * 80)
+    
+    def _save_end_to_end_results(self):
+        """Save detailed test results to JSON file"""
+        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+        filename = project_root / 'tests' / f'end_to_end_pipeline_results_{timestamp}.json'
+        
+        try:
+            comprehensive_results = {
+                "test_metadata": {
+                    "test_type": "end_to_end_pipeline",
+                    "timestamp": datetime.now().isoformat(),
+                    "session_start": self.start_time.isoformat(),
+                    "total_duration_seconds": (datetime.now() - self.start_time).total_seconds(),
+                    "total_tests": len(self.test_results),
+                    "successful_tests": len([r for r in self.test_results if r['success']]),
+                    "failed_tests": len([r for r in self.test_results if not r['success']])
+                },
+                "pipeline_results": self.test_results,
+                "component_status": {
+                    "user_prompt_processor": "operational",
+                    "retrieval_system": "operational", 
+                    "medical_generator": "operational",
+                    "med42_llm_client": "operational"
+                }
+            }
+            
+            with open(filename, 'w', encoding='utf-8') as f:
+                json.dump(comprehensive_results, f, indent=2, ensure_ascii=False)
+            
+            print(f"📁 End-to-end test results saved to: {filename}")
+            
+        except Exception as e:
+            logger.error(f"Failed to save test results: {e}")
+            print(f"⚠️ Failed to save test results: {e}")
+
+def main():
+    """Main execution function"""
+    print("🏥 OnCall.ai Complete End-to-End Pipeline Test")
+    print("Testing: User Input → UserPrompt → Retrieval → Generation")
+    print("=" * 70)
+    
+    # Initialize test suite
+    test_suite = EndToEndPipelineTest()
+    
+    # Initialize complete pipeline
+    test_suite.initialize_complete_pipeline()
+    
+    if not test_suite.components_initialized:
+        print("❌ Pipeline initialization failed. Cannot proceed with testing.")
+        return 1
+    
+    # Run scripted end-to-end tests
+    test_suite.run_scripted_end_to_end_tests()
+    
+    print(f"\n🎯 End-to-end testing completed!")
+    print("Next step: Create Gradio interface for interactive testing")
+    
+    return 0
+
+if __name__ == "__main__":
+    exit_code = main()
+    sys.exit(exit_code)

tests/test_multilevel_fallback_validation.py

@@ -0,0 +1,553 @@
+#!/usr/bin/env python3
+"""
+Multi-Level Fallback Validation Test Suite for OnCall.ai
+
+This test specifically validates the 5-level fallback mechanism:
+Level 1: Predefined Mapping (Fast Path)
+Level 2: Llama3-Med42-70B Extraction
+Level 3: Semantic Search Fallback
+Level 4: Medical Query Validation
+Level 5: Generic Medical Search
+
+Author: OnCall.ai Team
+Date: 2025-07-30
+"""
+
+import sys
+import os
+from pathlib import Path
+import logging
+import json
+import traceback
+from datetime import datetime
+from typing import Dict, List, Any, Optional
+
+# Add src directory to Python path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import our modules
+try:
+    from user_prompt import UserPromptProcessor
+    from retrieval import BasicRetrievalSystem
+    from llm_clients import llm_Med42_70BClient
+    from medical_conditions import CONDITION_KEYWORD_MAPPING
+except ImportError as e:
+    print(f"❌ Import Error: {e}")
+    print(f"Current working directory: {os.getcwd()}")
+    print(f"Python path: {sys.path}")
+    sys.exit(1)
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(),
+        logging.FileHandler(project_root / 'tests' / 'multilevel_fallback_test.log')
+    ]
+)
+logger = logging.getLogger(__name__)
+
+class MultilevelFallbackTest:
+    """Test suite specifically for the 5-level fallback mechanism"""
+    
+    def __init__(self):
+        """Initialize test suite"""
+        self.start_time = datetime.now()
+        self.results = []
+        self.components_initialized = False
+        
+        # Component references
+        self.llm_client = None
+        self.retrieval_system = None
+        self.user_prompt_processor = None
+        
+    def initialize_components(self):
+        """Initialize all pipeline components"""
+        print("🔧 Initializing Components for Multilevel Fallback Test...")
+        print("-" * 60)
+        
+        try:
+            # Initialize LLM client
+            print("1. Initializing Llama3-Med42-70B Client...")
+            self.llm_client = llm_Med42_70BClient()
+            print("   ✅ LLM client initialized")
+            
+            # Initialize retrieval system
+            print("2. Initializing Retrieval System...")
+            self.retrieval_system = BasicRetrievalSystem()
+            print("   ✅ Retrieval system initialized")
+            
+            # Initialize user prompt processor
+            print("3. Initializing User Prompt Processor...")
+            self.user_prompt_processor = UserPromptProcessor(
+                llm_client=self.llm_client,
+                retrieval_system=self.retrieval_system
+            )
+            print("   ✅ User prompt processor initialized")
+            
+            self.components_initialized = True
+            print("\n🎉 All components initialized successfully!")
+            
+        except Exception as e:
+            logger.error(f"Component initialization failed: {e}")
+            print(f"❌ Component initialization failed: {e}")
+            traceback.print_exc()
+            self.components_initialized = False
+    
+    def get_multilevel_test_cases(self) -> List[Dict[str, Any]]:
+        """Define test cases specifically targeting each fallback level"""
+        return [
+            # Level 1: Predefined Mapping Tests
+            {
+                "id": "level1_001",
+                "query": "acute myocardial infarction treatment",
+                "description": "Level 1: Direct predefined condition match",
+                "expected_level": 1,
+                "expected_condition": "acute myocardial infarction",
+                "expected_source": "predefined_mapping",
+                "category": "level1_predefined"
+            },
+            {
+                "id": "level1_002", 
+                "query": "how to manage acute stroke?",
+                "description": "Level 1: Predefined stroke condition",
+                "expected_level": 1,
+                "expected_condition": "acute stroke",
+                "expected_source": "predefined_mapping",
+                "category": "level1_predefined"
+            },
+            {
+                "id": "level1_003",
+                "query": "pulmonary embolism emergency protocol",
+                "description": "Level 1: Predefined PE condition",
+                "expected_level": 1,
+                "expected_condition": "pulmonary embolism",
+                "expected_source": "predefined_mapping",
+                "category": "level1_predefined"
+            },
+            
+            # Level 2: LLM Extraction Tests
+            {
+                "id": "level2_001",
+                "query": "patient with severe crushing chest pain radiating to left arm",
+                "description": "Level 2: Symptom-based query requiring LLM analysis",
+                "expected_level": 2,
+                "expected_condition": ["acute myocardial infarction", "acute coronary syndrome"],
+                "expected_source": "llm_extraction",
+                "category": "level2_llm"
+            },
+            {
+                "id": "level2_002",
+                "query": "sudden onset weakness on right side with speech difficulty",
+                "description": "Level 2: Neurological symptoms requiring LLM",
+                "expected_level": 2,
+                "expected_condition": ["acute stroke", "cerebrovascular accident"],
+                "expected_source": "llm_extraction",
+                "category": "level2_llm"
+            },
+            
+            # Level 3: Semantic Search Tests
+            {
+                "id": "level3_001",
+                "query": "emergency management of cardiovascular crisis",
+                "description": "Level 3: Generic medical terms requiring semantic search",
+                "expected_level": 3,
+                "expected_source": "semantic_search",
+                "category": "level3_semantic"
+            },
+            {
+                "id": "level3_002",
+                "query": "urgent neurological intervention protocols",
+                "description": "Level 3: Medical terminology requiring semantic fallback",
+                "expected_level": 3,
+                "expected_source": "semantic_search",
+                "category": "level3_semantic"
+            },
+            
+            # Level 4a: Non-Medical Query Rejection
+            {
+                "id": "level4a_001",
+                "query": "how to cook pasta properly?",
+                "description": "Level 4a: Non-medical query should be rejected",
+                "expected_level": 4,
+                "expected_result": "invalid_query",
+                "expected_source": "validation_rejection",
+                "category": "level4a_rejection"
+            },
+            {
+                "id": "level4a_002",
+                "query": "best programming language to learn in 2025",
+                "description": "Level 4a: Technology query should be rejected",
+                "expected_level": 4,
+                "expected_result": "invalid_query",
+                "expected_source": "validation_rejection",
+                "category": "level4a_rejection"
+            },
+            {
+                "id": "level4a_003",
+                "query": "weather forecast for tomorrow",
+                "description": "Level 4a: Weather query should be rejected",
+                "expected_level": 4,
+                "expected_result": "invalid_query",
+                "expected_source": "validation_rejection",
+                "category": "level4a_rejection"
+            },
+            
+            # Level 4b + 5: Obscure Medical Terms → Generic Search
+            {
+                "id": "level4b_001",
+                "query": "rare hematologic malignancy treatment approaches",
+                "description": "Level 4b→5: Obscure medical query passing validation to generic search",
+                "expected_level": 5,
+                "expected_condition": "generic medical query",
+                "expected_source": "generic_search",
+                "category": "level4b_to_5"
+            },
+            {
+                "id": "level4b_002",
+                "query": "idiopathic thrombocytopenic purpura management guidelines",
+                "description": "Level 4b→5: Rare condition requiring generic medical search",
+                "expected_level": 5,
+                "expected_condition": "generic medical query",
+                "expected_source": "generic_search",
+                "category": "level4b_to_5"
+            },
+            {
+                "id": "level4b_003",
+                "query": "necrotizing fasciitis surgical intervention protocols",
+                "description": "Level 4b→5: Rare emergency condition → generic search",
+                "expected_level": 5,
+                "expected_condition": "generic medical query",
+                "expected_source": "generic_search",
+                "category": "level4b_to_5"
+            }
+        ]
+    
+    def run_single_fallback_test(self, test_case: Dict[str, Any]) -> Dict[str, Any]:
+        """Execute a single fallback test case with level detection"""
+        test_id = test_case["id"]
+        query = test_case["query"]
+        
+        print(f"\n🔍 {test_id}: {test_case['description']}")
+        print(f"Query: '{query}'")
+        print(f"Expected Level: {test_case.get('expected_level', 'Unknown')}")
+        print("-" * 70)
+        
+        result = {
+            "test_id": test_id,
+            "test_case": test_case,
+            "timestamp": datetime.now().isoformat(),
+            "success": False,
+            "error": None,
+            "execution_time": 0,
+            "detected_level": None,
+            "condition_result": {}
+        }
+        
+        start_time = datetime.now()
+        
+        try:
+            # Execute condition extraction with level detection
+            print("🎯 Executing multilevel fallback...")
+            condition_start = datetime.now()
+            
+            condition_result = self.user_prompt_processor.extract_condition_keywords(query)
+            condition_time = (datetime.now() - condition_start).total_seconds()
+            
+            # Detect which level was used
+            detected_level = self._detect_fallback_level(condition_result)
+            
+            result["condition_result"] = condition_result
+            result["detected_level"] = detected_level
+            result["execution_time"] = condition_time
+            
+            print(f"   ✅ Detected Level: {detected_level}")
+            print(f"   Condition: {condition_result.get('condition', 'None')}")
+            print(f"   Emergency Keywords: {condition_result.get('emergency_keywords', 'None')}")
+            print(f"   Treatment Keywords: {condition_result.get('treatment_keywords', 'None')}")
+            print(f"   Execution Time: {condition_time:.3f}s")
+            
+            # Validate expected behavior
+            validation_result = self._validate_expected_behavior(test_case, detected_level, condition_result)
+            result.update(validation_result)
+            
+            if result["success"]:
+                print("   🎉 Test PASSED - Expected behavior achieved")
+            else:
+                print(f"   ⚠️  Test PARTIAL - {result.get('validation_message', 'Unexpected behavior')}")
+                
+        except Exception as e:
+            total_time = (datetime.now() - start_time).total_seconds()
+            result["execution_time"] = total_time
+            result["error"] = str(e)
+            result["traceback"] = traceback.format_exc()
+            
+            logger.error(f"Test {test_id} failed: {e}")
+            print(f"   ❌ Test FAILED: {e}")
+            
+        return result
+    
+    def _detect_fallback_level(self, condition_result: Dict[str, Any]) -> int:
+        """
+        Detect which fallback level was used based on the condition result.
+
+        Fallback levels:
+        0: No result or unknown fallback level.
+        1: Predefined Mapping (Fast Path) - The condition matches a predefined mapping.
+        2: Llama3-Med42-70B Extraction - The condition is extracted by the LLM.
+        3: Semantic Search Fallback - The result includes a semantic confidence score.
+        4: Medical Query Validation - The query is deemed invalid (e.g., 'invalid_query').
+        5: Generic Medical Search - The condition is identified as a generic medical query.
+
+        Args:
+            condition_result (Dict[str, Any]): The result of the condition extraction process.
+
+        Returns:
+            int: The detected fallback level (0-5).
+        """
+        if not condition_result:
+            return 0  # No result
+        
+        # Check for validation rejection (Level 4a)
+        if condition_result.get('type') == 'invalid_query':
+            return 4
+        
+        # Check for generic search (Level 5)
+        if condition_result.get('condition') == 'generic medical query':
+            return 5
+        
+        # Check for semantic search (Level 3)
+        if 'semantic_confidence' in condition_result:
+            return 3
+        
+        # Check for predefined mapping (Level 1)
+        condition = condition_result.get('condition', '')
+        if condition and condition in CONDITION_KEYWORD_MAPPING:
+            return 1
+        
+        # Otherwise assume LLM extraction (Level 2)
+        if condition:
+            return 2
+        
+        return 0  # Unknown
+    
+    def _validate_expected_behavior(self, test_case: Dict[str, Any], detected_level: int, 
+                                  condition_result: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate if the test behaved as expected"""
+        expected_level = test_case.get('expected_level')
+        validation_result = {
+            "level_match": detected_level == expected_level,
+            "condition_match": False,
+            "success": False,
+            "validation_message": ""
+        }
+        
+        # Check level match
+        if validation_result["level_match"]:
+            validation_result["validation_message"] += f"✅ Level {detected_level} as expected. "
+        else:
+            validation_result["validation_message"] += f"⚠️ Level {detected_level} != expected {expected_level}. "
+        
+        # Check condition/result match based on test type
+        if test_case["category"] == "level4a_rejection":
+            # Should be rejected
+            validation_result["condition_match"] = condition_result.get('type') == 'invalid_query'
+            if validation_result["condition_match"]:
+                validation_result["validation_message"] += "✅ Query correctly rejected. "
+            else:
+                validation_result["validation_message"] += "⚠️ Query should have been rejected. "
+                
+        elif test_case["category"] == "level4b_to_5":
+            # Should result in generic medical query
+            validation_result["condition_match"] = condition_result.get('condition') == 'generic medical query'
+            if validation_result["condition_match"]:
+                validation_result["validation_message"] += "✅ Generic medical search triggered. "
+            else:
+                validation_result["validation_message"] += "⚠️ Should trigger generic medical search. "
+                
+        else:
+            # Check expected condition
+            expected_conditions = test_case.get('expected_condition', [])
+            if isinstance(expected_conditions, str):
+                expected_conditions = [expected_conditions]
+            
+            actual_condition = condition_result.get('condition', '')
+            validation_result["condition_match"] = any(
+                expected.lower() in actual_condition.lower() 
+                for expected in expected_conditions
+            )
+            
+            if validation_result["condition_match"]:
+                validation_result["validation_message"] += f"✅ Condition '{actual_condition}' matches expected. "
+            else:
+                validation_result["validation_message"] += f"⚠️ Condition '{actual_condition}' != expected {expected_conditions}. "
+        
+        # Overall success
+        validation_result["success"] = validation_result["level_match"] or validation_result["condition_match"]
+        
+        return validation_result
+    
+    def run_all_fallback_tests(self):
+        """Execute all fallback tests and generate report"""
+        if not self.components_initialized:
+            print("❌ Cannot run tests: components not initialized")
+            return
+        
+        test_cases = self.get_multilevel_test_cases()
+        
+        print(f"\n🚀 Starting Multilevel Fallback Test Suite")
+        print(f"Total test cases: {len(test_cases)}")
+        print(f"Test started at: {self.start_time.strftime('%Y-%m-%d %H:%M:%S')}")
+        print("=" * 80)
+        
+        # Execute all tests
+        for test_case in test_cases:
+            result = self.run_single_fallback_test(test_case)
+            self.results.append(result)
+        
+        # Generate report
+        self.generate_fallback_report()
+        self.save_fallback_results()
+    
+    def generate_fallback_report(self):
+        """Generate detailed fallback analysis report"""
+        end_time = datetime.now()
+        total_duration = (end_time - self.start_time).total_seconds()
+        
+        successful_tests = [r for r in self.results if r['success']]
+        failed_tests = [r for r in self.results if not r['success']]
+        partial_tests = [r for r in self.results if not r['success'] and not r.get('error')]
+        
+        print("\n" + "=" * 80)
+        print("📊 MULTILEVEL FALLBACK TEST REPORT")
+        print("=" * 80)
+        
+        # Overall Statistics
+        print(f"🕐 Execution Summary:")
+        print(f"   Total duration: {total_duration:.3f}s")
+        print(f"   Average per test: {total_duration/len(self.results):.3f}s")
+        
+        print(f"\n📈 Test Results:")
+        print(f"   Total tests: {len(self.results)}")
+        print(f"   Passed: {len(successful_tests)} ✅")
+        print(f"   Partial: {len(partial_tests)} ⚠️")
+        print(f"   Failed: {len(failed_tests)} ❌")
+        print(f"   Success rate: {len(successful_tests)/len(self.results)*100:.1f}%")
+        
+        # Level Distribution Analysis
+        level_distribution = {}
+        level_performance = {}
+        
+        for result in self.results:
+            if not result.get('error'):
+                level = result.get('detected_level', 0)
+                level_distribution[level] = level_distribution.get(level, 0) + 1
+                
+                if level not in level_performance:
+                    level_performance[level] = []
+                level_performance[level].append(result['execution_time'])
+        
+        print(f"\n🎯 Level Distribution Analysis:")
+        for level in sorted(level_distribution.keys()):
+            count = level_distribution[level]
+            avg_time = sum(level_performance[level]) / len(level_performance[level])
+            level_name = {
+                1: "Predefined Mapping",
+                2: "LLM Extraction", 
+                3: "Semantic Search",
+                4: "Validation Rejection",
+                5: "Generic Search"
+            }.get(level, f"Unknown ({level})")
+            
+            print(f"   Level {level} ({level_name}): {count} tests, avg {avg_time:.3f}s")
+        
+        # Category Analysis
+        categories = {}
+        for result in self.results:
+            category = result['test_case']['category']
+            if category not in categories:
+                categories[category] = {'total': 0, 'passed': 0}
+            categories[category]['total'] += 1
+            if result['success']:
+                categories[category]['passed'] += 1
+        
+        print(f"\n📋 Category Analysis:")
+        for category, stats in categories.items():
+            success_rate = stats['passed'] / stats['total'] * 100
+            print(f"   {category}: {stats['passed']}/{stats['total']} ({success_rate:.1f}%)")
+        
+        # Detailed Results
+        print(f"\n📝 Detailed Test Results:")
+        for result in self.results:
+            test_case = result['test_case']
+            status = "✅ PASS" if result['success'] else ("❌ FAIL" if result.get('error') else "⚠️ PARTIAL")
+            
+            print(f"\n   {result['test_id']}: {status}")
+            print(f"      Query: '{test_case['query']}'")
+            print(f"      Expected Level: {test_case.get('expected_level', 'N/A')}")
+            print(f"      Detected Level: {result.get('detected_level', 'N/A')}")
+            print(f"      Condition: {result.get('condition_result', {}).get('condition', 'None')}")
+            print(f"      Time: {result['execution_time']:.3f}s")
+            
+            if result.get('validation_message'):
+                print(f"      Validation: {result['validation_message']}")
+            
+            if result.get('error'):
+                print(f"      Error: {result['error']}")
+        
+        print("\n" + "=" * 80)
+    
+    def save_fallback_results(self):
+        """Save detailed test results to JSON file"""
+        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+        filename = project_root / 'tests' / f'multilevel_fallback_results_{timestamp}.json'
+        
+        try:
+            comprehensive_results = {
+                "test_metadata": {
+                    "timestamp": datetime.now().isoformat(),
+                    "test_type": "multilevel_fallback_validation",
+                    "total_duration_seconds": (datetime.now() - self.start_time).total_seconds(),
+                    "total_tests": len(self.results),
+                    "passed_tests": len([r for r in self.results if r['success']]),
+                    "failed_tests": len([r for r in self.results if not r['success']])
+                },
+                "fallback_results": self.results
+            }
+            
+            with open(filename, 'w', encoding='utf-8') as f:
+                json.dump(comprehensive_results, f, indent=2, ensure_ascii=False)
+            
+            print(f"📁 Multilevel fallback results saved to: {filename}")
+            
+        except Exception as e:
+            logger.error(f"Failed to save test results: {e}")
+            print(f"⚠️ Failed to save test results: {e}")
+
+def main():
+    """Main execution function"""
+    print("🏥 OnCall.ai Multilevel Fallback Validation Test")
+    print("=" * 60)
+    
+    # Initialize test suite
+    test_suite = MultilevelFallbackTest()
+    
+    # Initialize components
+    test_suite.initialize_components()
+    
+    if not test_suite.components_initialized:
+        print("❌ Test suite initialization failed. Exiting.")
+        return 1
+    
+    # Run all fallback tests
+    test_suite.run_all_fallback_tests()
+    
+    return 0
+
+if __name__ == "__main__":
+    exit_code = main()
+    sys.exit(exit_code)

tests/test_retrieval.py

@@ -0,0 +1,206 @@
+"""
+Test suite for BasicRetrievalSystem
+This module tests the core retrieval functionality including:
+- System initialization
+- Basic search functionality
+- Deduplication logic
+- Result formatting
+"""
+
+import sys
+import os
+from pathlib import Path
+import logging
+
+print("\n=== Phase 1: Initializing Test Environment ===")
+# Add src to python path
+current_dir = Path(__file__).parent.resolve()
+project_root = current_dir.parent
+sys.path.append(str(project_root / "src"))
+
+print(f"• Current directory: {current_dir}")
+print(f"• Project root: {project_root}")
+print(f"• Python path added: {project_root / 'src'}")
+
+# Change working directory to project root for file access
+os.chdir(project_root)
+print(f"• Changed working directory to: {project_root}")
+
+from retrieval import BasicRetrievalSystem #type: ignore
+
+class TestRetrievalSystem:
+    """Test suite for basic retrieval system functionality"""
+    
+    def setup_class(self):
+        """Initialize test environment"""
+        print("\n=== Phase 2: Setting up Test Environment ===")
+        
+        # Setup logging to capture our logs
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(levelname)s:%(name)s:%(message)s',
+            handlers=[
+                logging.StreamHandler(),
+                logging.FileHandler('test_retrieval.log')
+            ]
+        )
+        
+        try:
+            print("• Initializing BasicRetrievalSystem...")
+            self.retrieval = BasicRetrievalSystem(embedding_dim=768)
+            print("✅ Retrieval system initialized successfully")
+            
+        except Exception as e:
+            print(f"❌ Failed to initialize retrieval system: {e}")
+            raise
+    
+    def test_system_initialization(self):
+        """Test system initialization components"""
+        print("\n=== Phase 3: System Initialization Test ===")
+        
+        print("• Checking embedding model...")
+        assert self.retrieval.embedding_model is not None, "Embedding model not loaded"
+        print("✓ Embedding model loaded")
+        
+        print("• Checking emergency index...")
+        assert self.retrieval.emergency_index is not None, "Emergency index not loaded"
+        print("✓ Emergency index loaded")
+        
+        print("• Checking treatment index...")
+        assert self.retrieval.treatment_index is not None, "Treatment index not loaded"
+        print("✓ Treatment index loaded")
+        
+        print("• Checking chunk data...")
+        assert len(self.retrieval.emergency_chunks) > 0, "Emergency chunks not loaded"
+        assert len(self.retrieval.treatment_chunks) > 0, "Treatment chunks not loaded"
+        print(f"✓ Emergency chunks: {len(self.retrieval.emergency_chunks)}")
+        print(f"✓ Treatment chunks: {len(self.retrieval.treatment_chunks)}")
+        
+        print("✅ System initialization test passed")
+    
+    def test_basic_search_functionality(self):
+        """Test basic search functionality with medical queries"""
+        print("\n=== Phase 4: Basic Search Functionality Test ===")
+        
+        test_queries = [
+            "What is the treatment for acute myocardial infarction?",
+            "How to manage chest pain in emergency?",
+            "Acute stroke treatment protocol"
+        ]
+        
+        for i, query in enumerate(test_queries, 1):
+            print(f"\n🔍 Test Query {i}/3: {query}")
+            
+            try:
+                results = self.retrieval.search(query)
+                
+                # Basic structure checks
+                assert "query" in results, "Query not in results"
+                assert "processed_results" in results, "Processed results not found"
+                assert "total_results" in results, "Total results count missing"
+                
+                processed_results = results["processed_results"]
+                print(f"• Results returned: {len(processed_results)}")
+                
+                # Check result format and display ALL results
+                for j, result in enumerate(processed_results, 1):  # Show ALL results
+                    assert "type" in result, f"Result {j} missing 'type' field"
+                    assert "text" in result, f"Result {j} missing 'text' field"
+                    assert "distance" in result, f"Result {j} missing 'distance' field"
+                    assert "chunk_id" in result, f"Result {j} missing 'chunk_id' field"
+                    
+                    print(f"  R-{j:2d} [{result['type']:9s}] (distance: {result['distance']:.3f}): {result['text'][:80]}...")
+                
+                print(f"✓ Query {i} completed successfully")
+                
+            except Exception as e:
+                print(f"❌ Query {i} failed: {e}")
+                raise
+        
+        print("\n✅ Basic search functionality test passed")
+    
+    def test_deduplication_logic(self):
+        """Test the text-based deduplication logic"""
+        print("\n=== Phase 5: Deduplication Logic Test ===")
+        
+        # Create test data with duplicate texts
+        test_results = [
+            {"text": "Sample text 1", "distance": 0.1, "type": "emergency", "chunk_id": 1},
+            {"text": "Sample text 1", "distance": 0.105, "type": "emergency", "chunk_id": 2},  # Duplicate text
+            {"text": "Sample text 3", "distance": 0.2, "type": "treatment", "chunk_id": 3},
+            {"text": "Sample text 4", "distance": 0.3, "type": "treatment", "chunk_id": 4}
+        ]
+        
+        print(f"• Original results: {len(test_results)}")
+        for i, result in enumerate(test_results, 1):
+            print(f"  Test-{i}: distance={result['distance']}, type={result['type']}")
+        
+        # Test deduplication
+        unique_results = self.retrieval._remove_duplicates(test_results)
+        
+        print(f"• After deduplication: {len(unique_results)}")
+        for i, result in enumerate(unique_results, 1):
+            print(f"  Kept-{i}: distance={result['distance']}, type={result['type']}")
+        
+        # Verify deduplication worked
+        assert len(unique_results) < len(test_results), "Deduplication should remove duplicate texts"
+        print("✓ Text-based deduplication working correctly")
+        
+        print("✅ Deduplication logic test passed")
+    
+    def test_result_statistics(self):
+        """Test result statistics and logging"""
+        print("\n=== Phase 6: Result Statistics Test ===")
+        
+        query = "Emergency cardiac arrest management"
+        print(f"• Testing with query: {query}")
+        
+        # Capture logs by running search
+        results = self.retrieval.search(query)
+        
+        # Verify we get statistics
+        assert "total_results" in results, "Total results missing"
+        assert "processing_info" in results, "Processing info missing"
+        
+        total_results = results["total_results"]
+        duplicates_removed = results["processing_info"]["duplicates_removed"]
+        
+        print(f"• Total results: {total_results}")
+        print(f"• Duplicates removed: {duplicates_removed}")
+        print("✓ Statistics logging working correctly")
+        
+        print("✅ Result statistics test passed")
+
+def main():
+    """Run all retrieval system tests"""
+    print("\n" + "="*60)
+    print("COMPREHENSIVE RETRIEVAL SYSTEM TEST SUITE")
+    print("="*60)
+    
+    test = TestRetrievalSystem()
+    
+    try:
+        test.setup_class()
+        test.test_system_initialization()
+        test.test_basic_search_functionality()
+        test.test_deduplication_logic()
+        test.test_result_statistics()
+        
+        print("\n" + "="*60)
+        print("🎉 ALL RETRIEVAL SYSTEM TESTS COMPLETED SUCCESSFULLY!")
+        print("="*60)
+        print("✅ System initialization validated")
+        print("✅ Basic search functionality confirmed")
+        print("✅ Text-based deduplication working")
+        print("✅ Result statistics and logging verified")
+        print("="*60)
+        
+    except Exception as e:
+        print("\n" + "="*60)
+        print("❌ RETRIEVAL SYSTEM TESTS FAILED!")
+        print(f"Error: {str(e)}")
+        print("="*60)
+        raise
+
+if __name__ == "__main__":
+    main() 

tests/test_user_prompt.py

@@ -0,0 +1,92 @@
+"""
+User Prompt Processor Test Suite
+
+Comprehensive unit tests for UserPromptProcessor class
+Ensures robust functionality across medical query scenarios.
+"""
+
+import pytest
+import sys
+from pathlib import Path
+
+# Dynamically add project root to Python path
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root / "src"))
+
+from user_prompt import UserPromptProcessor
+
+class TestUserPromptProcessor:
+    """Test suite for UserPromptProcessor functionality"""
+
+    def setup_method(self):
+        """Initialize test environment before each test method"""
+        self.processor = UserPromptProcessor()
+
+    def test_extract_condition_keywords_predefined(self):
+        """Test predefined condition extraction"""
+        query = "heart attack symptoms"
+        result = self.processor.extract_condition_keywords(query)
+        
+        assert result is not None
+        assert 'condition' in result
+        assert 'emergency_keywords' in result
+        assert 'treatment_keywords' in result
+
+    def test_handle_matching_failure_level1(self):
+        """Test loose keyword matching mechanism"""
+        test_queries = [
+            "urgent medical help",
+            "critical condition",
+            "severe symptoms"
+        ]
+        
+        for query in test_queries:
+            result = self.processor._handle_matching_failure_level1(query)
+            
+            assert result is not None
+            assert result['type'] == 'loose_keyword_match'
+            assert result['confidence'] == 0.5
+
+    def test_semantic_search_fallback(self):
+        """Verify semantic search fallback mechanism"""
+        test_queries = [
+            "how to manage chest pain",
+            "treatment for acute stroke",
+            "emergency cardiac care"
+        ]
+        
+        for query in test_queries:
+            result = self.processor._semantic_search_fallback(query)
+            
+            # Result can be None if no match found
+            if result is not None:
+                assert 'condition' in result
+                assert 'emergency_keywords' in result
+                assert 'treatment_keywords' in result
+
+    def test_validate_keywords(self):
+        """Test keyword validation functionality"""
+        valid_keywords = {
+            'emergency_keywords': 'urgent|critical',
+            'treatment_keywords': 'medication|therapy'
+        }
+        
+        invalid_keywords = {
+            'emergency_keywords': '',
+            'treatment_keywords': ''
+        }
+        
+        assert self.processor.validate_keywords(valid_keywords) is True
+        assert self.processor.validate_keywords(invalid_keywords) is False
+
+def main():
+    """Run comprehensive test suite with detailed reporting"""
+    print("\n" + "="*60)
+    print("OnCall.ai: User Prompt Processor Test Suite")
+    print("="*60)
+    
+    # Run pytest with verbose output
+    pytest.main([__file__, '-v', '--tb=short'])
+
+if __name__ == "__main__":
+    main() 

tests/test_userinput_userprompt_medical_condition_llm_retrieval.py

@@ -0,0 +1,479 @@
+#!/usr/bin/env python3
+"""
+Comprehensive Test Suite for OnCall.ai Medical Query Processing Pipeline
+
+This test validates the complete flow:
+User Input → UserPrompt Processing → Medical Condition Extraction → LLM Analysis → Retrieval
+
+Test Components:
+- UserPromptProcessor (condition extraction, keyword mapping)
+- MedicalConditions (predefined mappings, validation)
+- LLM Client (Llama3-Med42-70B condition extraction)
+- BasicRetrievalSystem (vector search, result processing)
+
+Author: OnCall.ai Team
+Date: 2025-07-30
+"""
+
+import sys
+import os
+from pathlib import Path
+import logging
+import json
+import traceback
+from datetime import datetime
+from typing import Dict, List, Any
+
+# Add src directory to Python path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import our modules
+try:
+    from user_prompt import UserPromptProcessor
+    from retrieval import BasicRetrievalSystem
+    from llm_clients import llm_Med42_70BClient
+    from medical_conditions import CONDITION_KEYWORD_MAPPING, validate_condition, get_condition_details
+except ImportError as e:
+    print(f"❌ Import Error: {e}")
+    print(f"Current working directory: {os.getcwd()}")
+    print(f"Python path: {sys.path}")
+    sys.exit(1)
+
+# Configure comprehensive logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(),
+        logging.FileHandler(project_root / 'tests' / 'pipeline_test.log')
+    ]
+)
+logger = logging.getLogger(__name__)
+
+class MedicalQueryPipelineTest:
+    """Comprehensive test suite for the medical query processing pipeline"""
+    
+    def __init__(self):
+        """Initialize test suite with all required components"""
+        self.start_time = datetime.now()
+        self.results = []
+        self.components_initialized = False
+        
+        # Component references
+        self.llm_client = None
+        self.retrieval_system = None
+        self.user_prompt_processor = None
+        
+    def initialize_components(self):
+        """Initialize all pipeline components with error handling"""
+        print("🔧 Initializing Pipeline Components...")
+        print("-" * 50)
+        
+        try:
+            # Initialize LLM client
+            print("1. Initializing Llama3-Med42-70B Client...")
+            self.llm_client = llm_Med42_70BClient()
+            print("   ✅ LLM client initialized successfully")
+            
+            # Initialize retrieval system
+            print("2. Initializing Retrieval System...")
+            self.retrieval_system = BasicRetrievalSystem()
+            print("   ✅ Retrieval system initialized successfully")
+            
+            # Initialize user prompt processor
+            print("3. Initializing User Prompt Processor...")
+            self.user_prompt_processor = UserPromptProcessor(
+                llm_client=self.llm_client,
+                retrieval_system=self.retrieval_system
+            )
+            print("   ✅ User prompt processor initialized successfully")
+            
+            self.components_initialized = True
+            print("\n🎉 All components initialized successfully!")
+            
+        except Exception as e:
+            logger.error(f"Component initialization failed: {e}")
+            print(f"❌ Component initialization failed: {e}")
+            traceback.print_exc()
+            self.components_initialized = False
+            
+    def get_test_queries(self) -> List[Dict[str, Any]]:
+        """Define comprehensive test queries with expected behavior"""
+        return [
+            {
+                "id": "test_001",
+                "query": "how to treat acute MI?",
+                "description": "Classic acute myocardial infarction query",
+                "expected_condition": "acute myocardial infarction",
+                "expected_mechanism": "predefined_mapping",
+                "category": "cardiac_emergency"
+            },
+            {
+                "id": "test_002", 
+                "query": "patient with severe chest pain and shortness of breath",
+                "description": "Symptoms-based query requiring LLM analysis",
+                "expected_condition": ["acute myocardial infarction", "pulmonary embolism", "acute coronary syndrome"],
+                "expected_mechanism": "llm_extraction",
+                "category": "cardiac_pulmonary"
+            },
+            {
+                "id": "test_003",
+                "query": "sudden neurological symptoms suggesting stroke",
+                "description": "Neurological emergency query",
+                "expected_condition": "acute stroke",
+                "expected_mechanism": "predefined_mapping",
+                "category": "neurological_emergency"
+            },
+            {
+                "id": "test_004",
+                "query": "acute stroke management protocol",
+                "description": "Protocol-specific stroke query",
+                "expected_condition": "acute stroke", 
+                "expected_mechanism": "predefined_mapping",
+                "category": "neurological_protocol"
+            },
+            {
+                "id": "test_005",
+                "query": "patient presenting with acute abdominal pain",
+                "description": "General symptom requiring LLM analysis",
+                "expected_condition": "unknown",
+                "expected_mechanism": "semantic_fallback",
+                "category": "general_symptom"
+            },
+            {
+                "id": "test_006",
+                "query": "pulmonary embolism treatment guidelines",
+                "description": "Specific condition with treatment focus",
+                "expected_condition": "pulmonary embolism",
+                "expected_mechanism": "predefined_mapping", 
+                "category": "pulmonary_emergency"
+            }
+        ]
+    
+    def run_single_test(self, test_case: Dict[str, Any]) -> Dict[str, Any]:
+        """Execute a single test case with comprehensive analysis"""
+        test_id = test_case["id"]
+        query = test_case["query"]
+        
+        print(f"\n🔍 {test_id}: {test_case['description']}")
+        print(f"Query: '{query}'")
+        print("-" * 60)
+        
+        result = {
+            "test_id": test_id,
+            "test_case": test_case,
+            "timestamp": datetime.now().isoformat(),
+            "success": False,
+            "error": None,
+            "execution_time": 0,
+            "steps": {}
+        }
+        
+        start_time = datetime.now()
+        
+        try:
+            # Step 1: Condition Extraction
+            print("Step 1: Extracting medical condition and keywords...")
+            condition_start = datetime.now()
+            
+            condition_result = self.user_prompt_processor.extract_condition_keywords(query)
+            condition_time = (datetime.now() - condition_start).total_seconds()
+            
+            result["steps"]["condition_extraction"] = {
+                "duration_seconds": condition_time,
+                "condition": condition_result.get('condition', ''),
+                "emergency_keywords": condition_result.get('emergency_keywords', ''),
+                "treatment_keywords": condition_result.get('treatment_keywords', ''),
+                "confidence": condition_result.get('confidence', 'unknown'),
+                "source": self._determine_extraction_source(condition_result)
+            }
+            
+            print(f"   Condition: {condition_result.get('condition', 'None')}")
+            print(f"   Emergency keywords: {condition_result.get('emergency_keywords', 'None')}")
+            print(f"   Treatment keywords: {condition_result.get('treatment_keywords', 'None')}")
+            print(f"   Source: {result['steps']['condition_extraction']['source']}")
+            print(f"   Duration: {condition_time:.3f}s")
+            
+            # Step 2: User Confirmation (Simulated)
+            print("\nStep 2: User confirmation process...")
+            confirmation_result = self.user_prompt_processor.handle_user_confirmation(condition_result)
+            
+            result["steps"]["user_confirmation"] = {
+                "confirmation_type": confirmation_result.get('type', 'unknown'),
+                "message_length": len(confirmation_result.get('message', '')),
+                "actionable": confirmation_result.get('type') == 'confirmation_needed'
+            }
+            
+            print(f"   Confirmation type: {confirmation_result.get('type', 'Unknown')}")
+            
+            # Step 3: Retrieval Execution
+            if condition_result.get('condition'):
+                print("\nStep 3: Executing retrieval...")
+                retrieval_start = datetime.now()
+                
+                # Construct search query
+                search_query = self._construct_search_query(condition_result)
+                
+                # Perform retrieval
+                retrieval_results = self.retrieval_system.search(search_query, top_k=5)
+                retrieval_time = (datetime.now() - retrieval_start).total_seconds()
+                
+                # Correctly count emergency and treatment results from processed_results
+                processed_results = retrieval_results.get('processed_results', [])
+                emergency_count = len([r for r in processed_results if r.get('type') == 'emergency'])
+                treatment_count = len([r for r in processed_results if r.get('type') == 'treatment'])
+                
+                result["steps"]["retrieval"] = {
+                    "duration_seconds": retrieval_time,
+                    "search_query": search_query,
+                    "total_results": retrieval_results.get('total_results', 0),
+                    "emergency_results": emergency_count,
+                    "treatment_results": treatment_count,
+                    "processed_results": len(processed_results),
+                    "duplicates_removed": retrieval_results.get('processing_info', {}).get('duplicates_removed', 0)
+                }
+                
+                print(f"   Search query: '{search_query}'")
+                print(f"   Total results: {result['steps']['retrieval']['total_results']}")
+                print(f"   Emergency results: {emergency_count}")
+                print(f"   Treatment results: {treatment_count}")
+                print(f"   Duration: {retrieval_time:.3f}s")
+                
+                # Analyze top results
+                if 'processed_results' in retrieval_results and retrieval_results['processed_results']:
+                    top_results = retrieval_results['processed_results'][:3]
+                    result["steps"]["top_results_analysis"] = []
+                    
+                    print(f"\n   Top {len(top_results)} results:")
+                    for i, res in enumerate(top_results, 1):
+                        analysis = {
+                            "rank": i,
+                            "type": res.get('type', 'unknown'),
+                            "distance": res.get('distance', 999),
+                            "text_length": len(res.get('text', '')),
+                            "has_matched_keywords": bool(res.get('matched', '')),
+                            "has_treatment_keywords": bool(res.get('matched_treatment', ''))
+                        }
+                        result["steps"]["top_results_analysis"].append(analysis)
+                        
+                        print(f"      {i}. Type: {analysis['type']}, Distance: {analysis['distance']:.4f}")
+                        print(f"         Text preview: {res.get('text', '')[:100]}...")
+                        if res.get('matched'):
+                            print(f"         Matched: {res.get('matched')}")
+                        if res.get('matched_treatment'):
+                            print(f"         Treatment: {res.get('matched_treatment')}")
+            
+            else:
+                print("\nStep 3: Skipping retrieval (no condition extracted)")
+                result["steps"]["retrieval"] = {
+                    "skipped": True,
+                    "reason": "no_condition_extracted"
+                }
+            
+            # Calculate total execution time
+            total_time = (datetime.now() - start_time).total_seconds()
+            result["execution_time"] = total_time
+            result["success"] = True
+            
+            print(f"\n✅ Test {test_id} completed successfully ({total_time:.3f}s)")
+            
+        except Exception as e:
+            total_time = (datetime.now() - start_time).total_seconds()
+            result["execution_time"] = total_time
+            result["error"] = str(e)
+            result["traceback"] = traceback.format_exc()
+            
+            logger.error(f"Test {test_id} failed: {e}")
+            print(f"❌ Test {test_id} failed: {e}")
+            
+        return result
+    
+    def _determine_extraction_source(self, condition_result: Dict) -> str:
+        """Determine how the condition was extracted"""
+        if condition_result.get('semantic_confidence') is not None:
+            return "semantic_search"
+        elif condition_result.get('generic_confidence') is not None:
+            return "generic_search"
+        elif condition_result.get('condition') in CONDITION_KEYWORD_MAPPING:
+            return "predefined_mapping"
+        else:
+            return "llm_extraction"
+    
+    def _construct_search_query(self, condition_result: Dict) -> str:
+        """Construct search query from condition result"""
+        emergency_kws = condition_result.get('emergency_keywords', '')
+        treatment_kws = condition_result.get('treatment_keywords', '')
+        
+        search_parts = []
+        if emergency_kws:
+            search_parts.append(emergency_kws)
+        if treatment_kws:
+            search_parts.append(treatment_kws)
+            
+        if search_parts:
+            return ' '.join(search_parts)
+        else:
+            return condition_result.get('condition', 'medical emergency')
+    
+    def run_all_tests(self):
+        """Execute all test cases and generate comprehensive report"""
+        if not self.components_initialized:
+            print("❌ Cannot run tests: components not initialized")
+            return
+        
+        test_cases = self.get_test_queries()
+        
+        print(f"\n🚀 Starting Comprehensive Pipeline Test")
+        print(f"Total test cases: {len(test_cases)}")
+        print(f"Test started at: {self.start_time.strftime('%Y-%m-%d %H:%M:%S')}")
+        print("=" * 80)
+        
+        # Execute all tests
+        for test_case in test_cases:
+            result = self.run_single_test(test_case)
+            self.results.append(result)
+        
+        # Generate comprehensive report
+        self.generate_test_report()
+        self.save_test_results()
+    
+    def generate_test_report(self):
+        """Generate detailed test report with statistics and analysis"""
+        end_time = datetime.now()
+        total_duration = (end_time - self.start_time).total_seconds()
+        
+        successful_tests = [r for r in self.results if r['success']]
+        failed_tests = [r for r in self.results if not r['success']]
+        
+        print("\n" + "=" * 80)
+        print("📊 COMPREHENSIVE TEST REPORT")
+        print("=" * 80)
+        
+        # Summary Statistics
+        print(f"🕐 Execution Summary:")
+        print(f"   Start time: {self.start_time.strftime('%Y-%m-%d %H:%M:%S')}")
+        print(f"   End time: {end_time.strftime('%Y-%m-%d %H:%M:%S')}")
+        print(f"   Total duration: {total_duration:.3f}s")
+        print(f"   Average per test: {total_duration/len(self.results):.3f}s")
+        
+        print(f"\n📈 Test Results:")
+        print(f"   Total tests: {len(self.results)}")
+        print(f"   Successful: {len(successful_tests)} ✅")
+        print(f"   Failed: {len(failed_tests)} ❌") 
+        print(f"   Success rate: {len(successful_tests)/len(self.results)*100:.1f}%")
+        
+        # Detailed Analysis
+        if successful_tests:
+            print(f"\n✅ Successful Tests Analysis:")
+            
+            # Analyze extraction sources
+            source_counts = {}
+            total_retrieval_time = 0
+            total_condition_time = 0
+            retrieval_count = 0
+            
+            for result in successful_tests:
+                if 'condition_extraction' in result['steps']:
+                    source = result['steps']['condition_extraction']['source']
+                    source_counts[source] = source_counts.get(source, 0) + 1
+                    total_condition_time += result['steps']['condition_extraction']['duration_seconds']
+                
+                if 'retrieval' in result['steps'] and not result['steps']['retrieval'].get('skipped'):
+                    total_retrieval_time += result['steps']['retrieval']['duration_seconds']
+                    retrieval_count += 1
+            
+            print(f"   Condition extraction sources:")
+            for source, count in source_counts.items():
+                print(f"     - {source}: {count} tests")
+            
+            print(f"   Performance metrics:")
+            print(f"     - Avg condition extraction: {total_condition_time/len(successful_tests):.3f}s")
+            if retrieval_count > 0:
+                print(f"     - Avg retrieval time: {total_retrieval_time/retrieval_count:.3f}s")
+            
+            # Individual test details
+            for result in successful_tests:
+                test_case = result['test_case']
+                print(f"\n   📋 {result['test_id']}: {test_case['description']}")
+                print(f"      Query: '{test_case['query']}'")
+                
+                if 'condition_extraction' in result['steps']:
+                    ce = result['steps']['condition_extraction']
+                    print(f"      Condition: {ce['condition']}")
+                    print(f"      Source: {ce['source']}")
+                
+                if 'retrieval' in result['steps'] and not result['steps']['retrieval'].get('skipped'):
+                    ret = result['steps']['retrieval']
+                    print(f"      Results: {ret['total_results']} total ({ret['emergency_results']} emergency, {ret['treatment_results']} treatment)")
+                
+                print(f"      Duration: {result['execution_time']:.3f}s")
+        
+        # Failed Tests Analysis
+        if failed_tests:
+            print(f"\n❌ Failed Tests Analysis:")
+            for result in failed_tests:
+                test_case = result['test_case']
+                print(f"   {result['test_id']}: {test_case['description']}")
+                print(f"      Error: {result['error']}")
+                print(f"      Duration: {result['execution_time']:.3f}s")
+        
+        print("\n" + "=" * 80)
+    
+    def save_test_results(self):
+        """Save detailed test results to JSON file"""
+        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+        filename = project_root / 'tests' / f'pipeline_test_results_{timestamp}.json'
+        
+        try:
+            comprehensive_results = {
+                "test_metadata": {
+                    "timestamp": datetime.now().isoformat(),
+                    "start_time": self.start_time.isoformat(),
+                    "total_duration_seconds": (datetime.now() - self.start_time).total_seconds(),
+                    "total_tests": len(self.results),
+                    "successful_tests": len([r for r in self.results if r['success']]),
+                    "failed_tests": len([r for r in self.results if not r['success']])
+                },
+                "test_results": self.results,
+                "component_versions": {
+                    "user_prompt_processor": "1.0.0",
+                    "retrieval_system": "1.0.0", 
+                    "llm_client": "1.0.0"
+                }
+            }
+            
+            with open(filename, 'w', encoding='utf-8') as f:
+                json.dump(comprehensive_results, f, indent=2, ensure_ascii=False)
+            
+            print(f"📁 Comprehensive test results saved to: {filename}")
+            
+        except Exception as e:
+            logger.error(f"Failed to save test results: {e}")
+            print(f"⚠️ Failed to save test results: {e}")
+
+def main():
+    """Main execution function"""
+    print("🏥 OnCall.ai Medical Query Processing Pipeline Test")
+    print("=" * 60)
+    
+    # Initialize test suite
+    test_suite = MedicalQueryPipelineTest()
+    
+    # Initialize components
+    test_suite.initialize_components()
+    
+    if not test_suite.components_initialized:
+        print("❌ Test suite initialization failed. Exiting.")
+        return 1
+    
+    # Run all tests
+    test_suite.run_all_tests()
+    
+    return 0
+
+if __name__ == "__main__":
+    exit_code = main()
+    sys.exit(exit_code)