Update app.py
Browse files
app.py
CHANGED
@@ -1,307 +1,570 @@
|
|
|
|
1 |
import gradio as gr
|
2 |
import torch
|
3 |
-
from transformers import
|
4 |
-
AutoModelForCausalLM,
|
5 |
-
AutoModelForImageTextToText,
|
6 |
-
AutoTokenizer,
|
7 |
-
AutoProcessor,
|
8 |
-
pipeline
|
9 |
-
)
|
10 |
from PIL import Image
|
11 |
-
import
|
12 |
-
import
|
13 |
-
|
14 |
-
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
# Configuration
|
23 |
-
MODEL_4B = "google/medgemma-4b-it"
|
24 |
-
MODEL_27B = "google/medgemma-27b-text-it"
|
25 |
-
|
26 |
-
class MedGemmaApp:
|
27 |
def __init__(self):
|
28 |
-
self.
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
def get_model_kwargs(self, use_quantization=True):
|
35 |
-
"""Get model configuration arguments"""
|
36 |
-
model_kwargs = {
|
37 |
-
"torch_dtype": torch.bfloat16,
|
38 |
-
"device_map": "auto",
|
39 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
40 |
|
41 |
-
|
42 |
-
|
43 |
-
model_kwargs["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True)
|
44 |
-
elif use_quantization and not QUANTIZATION_AVAILABLE:
|
45 |
-
print("β οΈ Quantization requested but bitsandbytes not available. Loading without quantization.")
|
46 |
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
# Clear previous model
|
57 |
-
if self.current_model is not None:
|
58 |
-
del self.current_model
|
59 |
-
del self.current_tokenizer
|
60 |
-
if self.current_processor:
|
61 |
-
del self.current_processor
|
62 |
-
if self.current_pipe:
|
63 |
-
del self.current_pipe
|
64 |
-
torch.cuda.empty_cache()
|
65 |
-
|
66 |
-
if model_choice == "4B (Multimodal)":
|
67 |
-
# Load multimodal model
|
68 |
-
self.current_model = AutoModelForImageTextToText.from_pretrained(
|
69 |
-
model_id, **model_kwargs
|
70 |
-
)
|
71 |
-
self.current_processor = AutoProcessor.from_pretrained(model_id)
|
72 |
-
self.model_type = "multimodal"
|
73 |
-
|
74 |
-
# Create pipeline for easier inference
|
75 |
-
self.current_pipe = pipeline(
|
76 |
-
"image-text-to-text",
|
77 |
-
model=self.current_model,
|
78 |
-
processor=self.current_processor,
|
79 |
-
)
|
80 |
-
self.current_pipe.model.generation_config.do_sample = False
|
81 |
-
|
82 |
-
else:
|
83 |
-
# Load text-only model
|
84 |
-
self.current_model = AutoModelForCausalLM.from_pretrained(
|
85 |
-
model_id, **model_kwargs
|
86 |
-
)
|
87 |
-
self.current_tokenizer = AutoTokenizer.from_pretrained(model_id)
|
88 |
-
self.model_type = "text"
|
89 |
-
|
90 |
-
# Create pipeline for easier inference
|
91 |
-
self.current_pipe = pipeline(
|
92 |
-
"text-generation",
|
93 |
-
model=self.current_model,
|
94 |
-
tokenizer=self.current_tokenizer,
|
95 |
-
)
|
96 |
-
self.current_pipe.model.generation_config.do_sample = False
|
97 |
-
|
98 |
-
return f"β
Successfully loaded {model_choice} model!"
|
99 |
-
|
100 |
-
except Exception as e:
|
101 |
-
return f"β Error loading model: {str(e)}"
|
102 |
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
|
107 |
-
|
108 |
-
|
109 |
-
|
110 |
-
|
111 |
-
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
|
118 |
-
|
119 |
-
|
120 |
-
|
121 |
-
|
122 |
-
|
123 |
-
|
124 |
-
|
125 |
-
|
126 |
-
return f"Error generating response: {str(e)}"
|
127 |
|
128 |
-
|
129 |
-
|
130 |
-
|
131 |
-
|
132 |
-
|
133 |
-
|
134 |
-
|
135 |
-
|
136 |
-
|
137 |
-
|
138 |
-
|
139 |
-
|
140 |
-
|
141 |
-
|
142 |
-
|
143 |
-
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
-
]
|
149 |
-
}
|
150 |
-
]
|
151 |
-
|
152 |
-
output = self.current_pipe(text=messages, max_new_tokens=300)
|
153 |
-
response = output[0]["generated_text"][-1]["content"]
|
154 |
-
|
155 |
-
return response
|
156 |
-
|
157 |
-
except Exception as e:
|
158 |
-
return f"Error analyzing image: {str(e)}"
|
159 |
|
160 |
-
#
|
161 |
-
|
162 |
|
163 |
-
|
164 |
-
|
165 |
-
|
166 |
-
# π₯ MedGemma Medical AI Assistant
|
167 |
|
168 |
-
|
169 |
-
|
170 |
-
|
|
|
171 |
|
172 |
-
|
173 |
-
|
|
|
174 |
|
175 |
-
|
176 |
-
|
177 |
-
|
178 |
-
|
179 |
-
|
180 |
-
|
181 |
-
|
182 |
-
|
183 |
-
|
184 |
-
|
185 |
-
|
186 |
-
|
187 |
-
|
188 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
189 |
)
|
190 |
-
|
191 |
-
|
192 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
193 |
|
194 |
-
|
195 |
-
|
196 |
-
|
197 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
198 |
|
199 |
-
|
200 |
-
|
201 |
-
|
202 |
-
|
203 |
-
|
204 |
-
|
205 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
206 |
|
207 |
-
|
208 |
-
|
209 |
-
|
210 |
-
|
211 |
-
)
|
212 |
|
213 |
-
|
214 |
-
text_input = gr.Textbox(
|
215 |
-
placeholder="Ask a medical question...",
|
216 |
-
label="Your Question",
|
217 |
-
scale=4
|
218 |
-
)
|
219 |
-
text_submit = gr.Button("Send", scale=1)
|
220 |
-
|
221 |
-
with gr.Column(scale=1):
|
222 |
gr.Markdown("""
|
223 |
-
|
224 |
-
-
|
225 |
-
-
|
226 |
-
-
|
227 |
-
-
|
|
|
228 |
""")
|
229 |
|
230 |
-
#
|
231 |
-
|
232 |
-
gr.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
233 |
|
234 |
-
|
235 |
-
with
|
236 |
-
|
237 |
-
|
238 |
-
|
239 |
-
|
240 |
-
|
241 |
-
|
242 |
-
|
243 |
-
|
244 |
-
|
245 |
-
|
246 |
-
|
247 |
-
|
248 |
-
value="Describe this X-ray",
|
249 |
-
label="Question about the image",
|
250 |
-
placeholder="What would you like to know about this image?"
|
251 |
-
)
|
252 |
-
|
253 |
-
image_submit = gr.Button("π Analyze Image", variant="primary")
|
254 |
-
|
255 |
-
with gr.Column(scale=2):
|
256 |
-
image_output = gr.Textbox(
|
257 |
-
label="Analysis Result",
|
258 |
-
lines=15,
|
259 |
-
placeholder="Upload an image and click 'Analyze Image' to see the AI's analysis..."
|
260 |
-
)
|
261 |
-
|
262 |
-
# Event handlers
|
263 |
-
load_btn.click(
|
264 |
-
fn=app.load_model,
|
265 |
-
inputs=[model_choice, use_quantization],
|
266 |
-
outputs=[model_status]
|
267 |
-
)
|
268 |
-
|
269 |
-
def respond_text(message, history, system_instruction):
|
270 |
-
if message.strip() == "":
|
271 |
-
return history, ""
|
272 |
|
273 |
-
|
274 |
-
|
275 |
-
|
276 |
-
|
277 |
-
|
278 |
-
|
279 |
-
|
280 |
-
|
281 |
-
|
282 |
-
|
283 |
-
|
284 |
-
|
285 |
-
|
286 |
-
|
287 |
-
|
288 |
-
|
289 |
-
|
290 |
-
|
291 |
-
|
292 |
-
|
293 |
-
|
294 |
-
|
295 |
-
|
296 |
-
|
297 |
-
|
298 |
-
|
299 |
-
|
300 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
301 |
|
302 |
-
|
303 |
-
Always consult qualified healthcare professionals for medical advice.
|
304 |
-
""")
|
305 |
|
|
|
306 |
if __name__ == "__main__":
|
307 |
-
demo
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# app.py - Medical AI using LLaVA (Large Language and Vision Assistant)
|
2 |
import gradio as gr
|
3 |
import torch
|
4 |
+
from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration
|
|
|
|
|
|
|
|
|
|
|
|
|
5 |
from PIL import Image
|
6 |
+
import logging
|
7 |
+
from collections import defaultdict, Counter
|
8 |
+
import time
|
9 |
+
|
10 |
+
# Configure logging
|
11 |
+
logging.basicConfig(level=logging.INFO)
|
12 |
+
logger = logging.getLogger(__name__)
|
13 |
+
|
14 |
+
# Usage tracking
|
15 |
+
class UsageTracker:
|
|
|
|
|
|
|
|
|
|
|
|
|
16 |
def __init__(self):
|
17 |
+
self.stats = {
|
18 |
+
'total_analyses': 0,
|
19 |
+
'successful_analyses': 0,
|
20 |
+
'failed_analyses': 0,
|
21 |
+
'average_processing_time': 0.0,
|
22 |
+
'question_types': Counter()
|
|
|
|
|
|
|
|
|
|
|
23 |
}
|
24 |
+
|
25 |
+
def log_analysis(self, success, duration, question_type=None):
|
26 |
+
self.stats['total_analyses'] += 1
|
27 |
+
if success:
|
28 |
+
self.stats['successful_analyses'] += 1
|
29 |
+
else:
|
30 |
+
self.stats['failed_analyses'] += 1
|
31 |
|
32 |
+
total_time = self.stats['average_processing_time'] * (self.stats['total_analyses'] - 1)
|
33 |
+
self.stats['average_processing_time'] = (total_time + duration) / self.stats['total_analyses']
|
|
|
|
|
|
|
34 |
|
35 |
+
if question_type:
|
36 |
+
self.stats['question_types'][question_type] += 1
|
37 |
+
|
38 |
+
# Rate limiting
|
39 |
+
class RateLimiter:
|
40 |
+
def __init__(self, max_requests_per_hour=30):
|
41 |
+
self.max_requests_per_hour = max_requests_per_hour
|
42 |
+
self.requests = defaultdict(list)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
43 |
|
44 |
+
def is_allowed(self, user_id="default"):
|
45 |
+
current_time = time.time()
|
46 |
+
hour_ago = current_time - 3600
|
47 |
+
self.requests[user_id] = [req_time for req_time in self.requests[user_id] if req_time > hour_ago]
|
48 |
+
if len(self.requests[user_id]) < self.max_requests_per_hour:
|
49 |
+
self.requests[user_id].append(current_time)
|
50 |
+
return True
|
51 |
+
return False
|
52 |
+
|
53 |
+
# Initialize components
|
54 |
+
usage_tracker = UsageTracker()
|
55 |
+
rate_limiter = RateLimiter()
|
56 |
+
|
57 |
+
# Model configuration - Using LLaVA-Next (latest version)
|
58 |
+
MODEL_ID = "llava-hf/llava-v1.6-mistral-7b-hf"
|
59 |
+
|
60 |
+
# Global variables
|
61 |
+
model = None
|
62 |
+
processor = None
|
63 |
+
|
64 |
+
def load_llava():
|
65 |
+
"""Load LLaVA model for medical analysis"""
|
66 |
+
global model, processor
|
|
|
67 |
|
68 |
+
try:
|
69 |
+
logger.info(f"Loading LLaVA model: {MODEL_ID}")
|
70 |
+
|
71 |
+
# Load processor
|
72 |
+
processor = LlavaNextProcessor.from_pretrained(MODEL_ID)
|
73 |
+
|
74 |
+
# Load model with appropriate settings
|
75 |
+
model = LlavaNextForConditionalGeneration.from_pretrained(
|
76 |
+
MODEL_ID,
|
77 |
+
torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
|
78 |
+
device_map="auto" if torch.cuda.is_available() else None,
|
79 |
+
low_cpu_mem_usage=True
|
80 |
+
)
|
81 |
+
|
82 |
+
logger.info("β
LLaVA model loaded successfully!")
|
83 |
+
return True
|
84 |
+
|
85 |
+
except Exception as e:
|
86 |
+
logger.error(f"β Error loading LLaVA: {str(e)}")
|
87 |
+
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
88 |
|
89 |
+
# Load model at startup
|
90 |
+
llava_ready = load_llava()
|
91 |
|
92 |
+
def analyze_medical_image_llava(image, clinical_question, patient_history=""):
|
93 |
+
"""Analyze medical image using LLaVA"""
|
94 |
+
start_time = time.time()
|
|
|
95 |
|
96 |
+
# Rate limiting
|
97 |
+
if not rate_limiter.is_allowed():
|
98 |
+
usage_tracker.log_analysis(False, time.time() - start_time)
|
99 |
+
return "β οΈ Rate limit exceeded. Please wait before trying again."
|
100 |
|
101 |
+
if not llava_ready or model is None:
|
102 |
+
usage_tracker.log_analysis(False, time.time() - start_time)
|
103 |
+
return "β LLaVA model not loaded. Please refresh the page and wait for model loading."
|
104 |
|
105 |
+
if image is None:
|
106 |
+
return "β οΈ Please upload a medical image first."
|
107 |
+
|
108 |
+
if not clinical_question.strip():
|
109 |
+
return "β οΈ Please provide a clinical question."
|
110 |
+
|
111 |
+
try:
|
112 |
+
logger.info("Starting LLaVA medical analysis...")
|
113 |
+
|
114 |
+
# Prepare comprehensive medical prompt
|
115 |
+
medical_prompt = f"""You are a highly skilled medical AI assistant with expertise in medical image analysis. You have extensive knowledge in:
|
116 |
+
|
117 |
+
- **Radiology**: X-rays, CT scans, MRI, ultrasound interpretation
|
118 |
+
- **Pathology**: Histological analysis, tissue examination, cellular patterns
|
119 |
+
- **Dermatology**: Skin lesions, rashes, dermatological conditions
|
120 |
+
- **Ophthalmology**: Retinal imaging, eye examinations, ocular pathology
|
121 |
+
- **General Medical Imaging**: Cross-sectional anatomy, normal variants, pathological findings
|
122 |
+
|
123 |
+
**Patient Information:**
|
124 |
+
{f"Patient History: {patient_history}" if patient_history.strip() else "No specific patient history provided"}
|
125 |
+
|
126 |
+
**Clinical Question:** {clinical_question}
|
127 |
+
|
128 |
+
**Instructions:**
|
129 |
+
Please provide a comprehensive medical analysis of this image following this structure:
|
130 |
+
|
131 |
+
1. **IMAGE QUALITY ASSESSMENT**
|
132 |
+
- Technical adequacy of the image
|
133 |
+
- Any artifacts or limitations
|
134 |
+
- Overall diagnostic quality
|
135 |
+
|
136 |
+
2. **SYSTEMATIC OBSERVATION**
|
137 |
+
- Describe what you see in detail
|
138 |
+
- Identify anatomical structures visible
|
139 |
+
- Note any normal findings
|
140 |
+
|
141 |
+
3. **ABNORMAL FINDINGS**
|
142 |
+
- Identify any pathological changes
|
143 |
+
- Describe abnormalities in detail
|
144 |
+
- Note their location and characteristics
|
145 |
+
|
146 |
+
4. **CLINICAL SIGNIFICANCE**
|
147 |
+
- Explain the importance of findings
|
148 |
+
- Relate to potential diagnoses
|
149 |
+
- Discuss clinical implications
|
150 |
+
|
151 |
+
5. **DIFFERENTIAL DIAGNOSIS**
|
152 |
+
- List possible conditions
|
153 |
+
- Explain reasoning for each
|
154 |
+
- Prioritize based on imaging findings
|
155 |
+
|
156 |
+
6. **RECOMMENDATIONS**
|
157 |
+
- Suggest additional imaging if needed
|
158 |
+
- Recommend clinical correlation
|
159 |
+
- Advise on follow-up or further evaluation
|
160 |
+
|
161 |
+
Please be thorough, educational, and professional in your analysis. Always emphasize that this is for educational purposes and requires professional medical validation."""
|
162 |
+
|
163 |
+
# Prepare conversation for LLaVA
|
164 |
+
conversation = [
|
165 |
+
{
|
166 |
+
"role": "user",
|
167 |
+
"content": [
|
168 |
+
{"type": "text", "text": medical_prompt},
|
169 |
+
{"type": "image", "image": image}
|
170 |
+
]
|
171 |
+
}
|
172 |
+
]
|
173 |
+
|
174 |
+
# Apply chat template
|
175 |
+
prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
|
176 |
+
|
177 |
+
# Process inputs
|
178 |
+
inputs = processor(prompt, image, return_tensors='pt')
|
179 |
+
|
180 |
+
# Move to appropriate device
|
181 |
+
if torch.cuda.is_available() and hasattr(model, 'device'):
|
182 |
+
inputs = {k: v.to(model.device) for k, v in inputs.items()}
|
183 |
+
|
184 |
+
# Generate response
|
185 |
+
logger.info("Generating comprehensive medical analysis...")
|
186 |
+
with torch.inference_mode():
|
187 |
+
output = model.generate(
|
188 |
+
**inputs,
|
189 |
+
max_new_tokens=2000,
|
190 |
+
do_sample=True,
|
191 |
+
temperature=0.2, # Lower temperature for more focused medical analysis
|
192 |
+
top_p=0.9,
|
193 |
+
repetition_penalty=1.1,
|
194 |
+
pad_token_id=processor.tokenizer.eos_token_id
|
195 |
)
|
196 |
+
|
197 |
+
# Decode response
|
198 |
+
generated_text = processor.decode(output[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True)
|
199 |
+
|
200 |
+
# Clean up response
|
201 |
+
response = generated_text.strip()
|
202 |
+
|
203 |
+
# Format the response with medical structure
|
204 |
+
formatted_response = f"""# π₯ **LLaVA Medical Image Analysis**
|
205 |
+
|
206 |
+
## **Clinical Question:** {clinical_question}
|
207 |
+
{f"## **Patient History:** {patient_history}" if patient_history.strip() else ""}
|
208 |
+
|
209 |
+
---
|
210 |
+
|
211 |
+
## π **Comprehensive Medical Analysis**
|
212 |
+
|
213 |
+
{response}
|
214 |
+
|
215 |
+
---
|
216 |
+
|
217 |
+
## π **Summary and Clinical Correlation**
|
218 |
+
|
219 |
+
**Key Points:**
|
220 |
+
- This analysis provides a systematic approach to medical image interpretation
|
221 |
+
- All findings should be correlated with clinical presentation and patient history
|
222 |
+
- The AI assessment serves as an educational tool and decision support aid
|
223 |
+
|
224 |
+
**Clinical Workflow:**
|
225 |
+
1. **Review** the systematic analysis above
|
226 |
+
2. **Correlate** findings with patient symptoms and history
|
227 |
+
3. **Consult** with appropriate medical specialists as needed
|
228 |
+
4. **Document** findings in the patient's medical record
|
229 |
+
5. **Follow up** with recommended additional studies if indicated
|
230 |
+
|
231 |
+
**Educational Value:**
|
232 |
+
This analysis demonstrates structured medical image interpretation methodology and clinical reasoning processes used in healthcare settings.
|
233 |
+
"""
|
234 |
+
|
235 |
+
# Add comprehensive medical disclaimer
|
236 |
+
disclaimer = """
|
237 |
+
---
|
238 |
+
## β οΈ **IMPORTANT MEDICAL DISCLAIMER**
|
239 |
+
|
240 |
+
**FOR EDUCATIONAL AND RESEARCH PURPOSES ONLY**
|
241 |
+
|
242 |
+
- **Not a Medical Diagnosis**: This AI analysis does not constitute a medical diagnosis, treatment recommendation, or professional medical advice
|
243 |
+
- **Professional Review Required**: All findings must be validated by qualified healthcare professionals
|
244 |
+
- **Emergency Situations**: For urgent medical concerns, contact emergency services immediately (911 in US)
|
245 |
+
- **Clinical Correlation**: AI findings must be correlated with clinical examination and patient history
|
246 |
+
- **Liability**: This system is not intended for clinical decision-making and users assume all responsibility
|
247 |
+
- **Educational Tool**: Designed for medical education, training, and research applications only
|
248 |
+
- **Data Privacy**: Do not upload images containing patient identifiable information
|
249 |
+
|
250 |
+
**Always consult qualified healthcare professionals for medical diagnosis and treatment decisions.**
|
251 |
+
|
252 |
+
---
|
253 |
+
**Powered by**: LLaVA (Large Language and Vision Assistant) | **Model**: {MODEL_ID}
|
254 |
+
"""
|
255 |
+
|
256 |
+
# Log successful analysis
|
257 |
+
duration = time.time() - start_time
|
258 |
+
question_type = classify_question(clinical_question)
|
259 |
+
usage_tracker.log_analysis(True, duration, question_type)
|
260 |
+
|
261 |
+
logger.info("β
LLaVA medical analysis completed successfully")
|
262 |
+
return formatted_response + disclaimer
|
263 |
+
|
264 |
+
except Exception as e:
|
265 |
+
duration = time.time() - start_time
|
266 |
+
usage_tracker.log_analysis(False, duration)
|
267 |
+
logger.error(f"β LLaVA analysis error: {str(e)}")
|
268 |
+
|
269 |
+
if "memory" in str(e).lower() or "cuda" in str(e).lower():
|
270 |
+
return "β **Memory Error**: The model requires more memory. Try using a smaller image or upgrading to GPU hardware."
|
271 |
+
else:
|
272 |
+
return f"β **Analysis Failed**: {str(e)}\n\nPlease try again with a different image or contact support if the issue persists."
|
273 |
+
|
274 |
+
def classify_question(question):
|
275 |
+
"""Classify clinical question type"""
|
276 |
+
question_lower = question.lower()
|
277 |
+
if any(word in question_lower for word in ['describe', 'findings', 'observe', 'see']):
|
278 |
+
return 'descriptive'
|
279 |
+
elif any(word in question_lower for word in ['diagnosis', 'differential', 'condition', 'disease']):
|
280 |
+
return 'diagnostic'
|
281 |
+
elif any(word in question_lower for word in ['abnormal', 'pathology', 'lesion', 'mass']):
|
282 |
+
return 'pathological'
|
283 |
+
elif any(word in question_lower for word in ['analyze', 'assess', 'evaluate', 'review']):
|
284 |
+
return 'analytical'
|
285 |
+
else:
|
286 |
+
return 'general'
|
287 |
+
|
288 |
+
def get_usage_stats():
|
289 |
+
"""Get comprehensive usage statistics"""
|
290 |
+
stats = usage_tracker.stats
|
291 |
+
if stats['total_analyses'] == 0:
|
292 |
+
return "π **Usage Statistics**\n\nNo analyses performed yet."
|
293 |
|
294 |
+
success_rate = (stats['successful_analyses'] / stats['total_analyses']) * 100
|
295 |
+
|
296 |
+
return f"""π **LLaVA Medical AI Usage Statistics**
|
297 |
+
|
298 |
+
**Performance Metrics:**
|
299 |
+
- **Total Analyses**: {stats['total_analyses']}
|
300 |
+
- **Success Rate**: {success_rate:.1f}%
|
301 |
+
- **Average Processing Time**: {stats['average_processing_time']:.2f} seconds
|
302 |
+
- **Failed Analyses**: {stats['failed_analyses']}
|
303 |
+
|
304 |
+
**Question Type Distribution:**
|
305 |
+
{chr(10).join([f"- **{qtype.title()}**: {count} ({count/stats['total_analyses']*100:.1f}%)" for qtype, count in stats['question_types'].most_common()])}
|
306 |
+
|
307 |
+
**System Information:**
|
308 |
+
- **Model**: LLaVA-v1.6-Mistral-7B
|
309 |
+
- **Capabilities**: Medical image analysis and clinical reasoning
|
310 |
+
- **Device**: {'GPU' if torch.cuda.is_available() else 'CPU'}
|
311 |
+
- **Status**: {'π’ Operational' if llava_ready else 'π΄ Offline'}
|
312 |
+
"""
|
313 |
+
|
314 |
+
# Create comprehensive Gradio interface
|
315 |
+
def create_interface():
|
316 |
+
with gr.Blocks(
|
317 |
+
title="LLaVA Medical Image Analysis",
|
318 |
+
theme=gr.themes.Soft(),
|
319 |
+
css="""
|
320 |
+
.gradio-container { max-width: 1400px !important; }
|
321 |
+
.disclaimer { background-color: #fef2f2; border: 1px solid #fecaca; border-radius: 8px; padding: 16px; margin: 16px 0; }
|
322 |
+
.success { background-color: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin: 16px 0; }
|
323 |
+
.warning { background-color: #fffbeb; border: 1px solid #fed7aa; border-radius: 8px; padding: 16px; margin: 16px 0; }
|
324 |
+
"""
|
325 |
+
) as demo:
|
326 |
+
|
327 |
+
# Header
|
328 |
+
gr.Markdown("""
|
329 |
+
# π₯ LLaVA Medical Image Analysis
|
330 |
+
|
331 |
+
**Advanced Medical AI powered by LLaVA (Large Language and Vision Assistant)**
|
332 |
+
|
333 |
+
**Specialized Medical Capabilities:**
|
334 |
+
π« **Radiology** β’ π¬ **Pathology** β’ π©Ί **Dermatology** β’ ποΈ **Ophthalmology** β’ π§ **Clinical Reasoning**
|
335 |
+
""")
|
336 |
+
|
337 |
+
# Status display
|
338 |
+
if llava_ready:
|
339 |
+
gr.Markdown("""
|
340 |
+
<div class="success">
|
341 |
+
β
<strong>LLAVA MEDICAL AI READY</strong><br>
|
342 |
+
LLaVA vision-language model loaded successfully. Ready for comprehensive medical image analysis with clinical reasoning.
|
343 |
+
</div>
|
344 |
+
""")
|
345 |
+
else:
|
346 |
+
gr.Markdown("""
|
347 |
+
<div class="warning">
|
348 |
+
β οΈ <strong>MODEL LOADING IN PROGRESS</strong><br>
|
349 |
+
LLaVA model is loading. This may take a few minutes. Please wait and refresh the page.
|
350 |
+
</div>
|
351 |
+
""")
|
352 |
+
|
353 |
+
# Medical disclaimer
|
354 |
+
gr.Markdown("""
|
355 |
+
<div class="disclaimer">
|
356 |
+
β οΈ <strong>CRITICAL MEDICAL DISCLAIMER</strong><br>
|
357 |
+
This AI tool provides <strong>educational medical analysis only</strong>. It is NOT a substitute for professional medical diagnosis.
|
358 |
+
<br><br>
|
359 |
+
<strong>Do NOT upload real patient data or PHI.</strong> Always consult qualified healthcare professionals for medical decisions.
|
360 |
+
</div>
|
361 |
+
""")
|
362 |
+
|
363 |
+
with gr.Row():
|
364 |
+
# Left column - Main interface
|
365 |
+
with gr.Column(scale=2):
|
366 |
+
with gr.Row():
|
367 |
+
with gr.Column():
|
368 |
+
gr.Markdown("## π€ Medical Image Upload")
|
369 |
+
image_input = gr.Image(
|
370 |
+
label="Upload Medical Image",
|
371 |
+
type="pil",
|
372 |
+
height=350,
|
373 |
+
sources=["upload", "clipboard"]
|
374 |
+
)
|
375 |
+
|
376 |
+
with gr.Column():
|
377 |
+
gr.Markdown("## π¬ Clinical Information")
|
378 |
+
clinical_question = gr.Textbox(
|
379 |
+
label="Clinical Question *",
|
380 |
+
placeholder="Examples:\nβ’ Analyze this chest X-ray comprehensively\nβ’ What pathological findings are visible in this image?\nβ’ Provide differential diagnosis based on these imaging findings\nβ’ Describe abnormalities and their clinical significance\nβ’ Evaluate this medical image systematically",
|
381 |
+
lines=5,
|
382 |
+
max_lines=8
|
383 |
+
)
|
384 |
+
|
385 |
+
patient_history = gr.Textbox(
|
386 |
+
label="Patient History & Clinical Context (Optional)",
|
387 |
+
placeholder="e.g., 58-year-old female presenting with chest pain and shortness of breath. History of hypertension and smoking. Recent onset of symptoms.",
|
388 |
+
lines=3,
|
389 |
+
max_lines=5
|
390 |
+
)
|
391 |
+
|
392 |
+
with gr.Row():
|
393 |
+
clear_btn = gr.Button("ποΈ Clear All", variant="secondary")
|
394 |
+
analyze_btn = gr.Button("π Analyze with LLaVA", variant="primary", size="lg")
|
395 |
+
|
396 |
+
gr.Markdown("## π LLaVA Medical Analysis Results")
|
397 |
+
output = gr.Textbox(
|
398 |
+
label="Comprehensive Medical Analysis",
|
399 |
+
lines=30,
|
400 |
+
max_lines=50,
|
401 |
+
show_copy_button=True,
|
402 |
+
placeholder="Upload a medical image and provide a clinical question to receive comprehensive AI-powered medical analysis..." if llava_ready else "LLaVA model is loading. Please wait and refresh the page."
|
403 |
+
)
|
404 |
|
405 |
+
# Right column - System info and controls
|
406 |
+
with gr.Column(scale=1):
|
407 |
+
gr.Markdown("## βΉοΈ System Status")
|
408 |
+
|
409 |
+
model_status = "β
Ready" if llava_ready else "π Loading"
|
410 |
+
device_info = "GPU" if torch.cuda.is_available() else "CPU"
|
411 |
+
|
412 |
+
gr.Markdown(f"""
|
413 |
+
**Model Status:** {model_status}
|
414 |
+
**AI Model:** LLaVA-v1.6-Mistral-7B
|
415 |
+
**Device:** {device_info}
|
416 |
+
**Capabilities:** Medical image analysis + clinical reasoning
|
417 |
+
**Context Length:** 32K tokens
|
418 |
+
**Rate Limit:** 30 requests/hour
|
419 |
+
""")
|
420 |
+
|
421 |
+
gr.Markdown("## π Usage Analytics")
|
422 |
+
stats_display = gr.Markdown("")
|
423 |
+
refresh_stats_btn = gr.Button("π Refresh Statistics", size="sm")
|
424 |
+
|
425 |
+
if llava_ready:
|
426 |
+
gr.Markdown("## π― Quick Clinical Examples")
|
427 |
|
428 |
+
radiology_btn = gr.Button("π« Chest X-ray Analysis", size="sm")
|
429 |
+
pathology_btn = gr.Button("π¬ Pathology Review", size="sm")
|
430 |
+
dermatology_btn = gr.Button("π©Ί Skin Lesion Analysis", size="sm")
|
431 |
+
differential_btn = gr.Button("π§ Differential Diagnosis", size="sm")
|
|
|
432 |
|
433 |
+
gr.Markdown("## π₯ Medical Specialties")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
434 |
gr.Markdown("""
|
435 |
+
**LLaVA excels in:**
|
436 |
+
- Radiology interpretation
|
437 |
+
- Pathological analysis
|
438 |
+
- Dermatological assessment
|
439 |
+
- Ophthalmological evaluation
|
440 |
+
- Clinical reasoning & education
|
441 |
""")
|
442 |
|
443 |
+
# Comprehensive example cases
|
444 |
+
if llava_ready:
|
445 |
+
with gr.Accordion("π Sample Medical Cases & Examples", open=False):
|
446 |
+
examples = gr.Examples(
|
447 |
+
examples=[
|
448 |
+
[
|
449 |
+
"https://upload.wikimedia.org/wikipedia/commons/c/c8/Chest_Xray_PA_3-8-2010.png",
|
450 |
+
"Please perform a comprehensive systematic analysis of this chest X-ray. Evaluate image quality, assess cardiac silhouette, examine lung fields bilaterally, review mediastinal structures, and identify any pathological findings. Provide differential diagnosis if abnormalities are present.",
|
451 |
+
"Adult patient presenting with acute onset chest pain and shortness of breath. No significant past medical history."
|
452 |
+
],
|
453 |
+
[
|
454 |
+
None,
|
455 |
+
"Analyze this medical image systematically. Describe normal anatomical structures, identify any abnormal findings, assess clinical significance, and provide appropriate differential diagnoses based on imaging characteristics.",
|
456 |
+
"Patient with acute presentation requiring medical imaging evaluation"
|
457 |
+
],
|
458 |
+
[
|
459 |
+
None,
|
460 |
+
"What pathological changes are visible in this medical image? Please provide detailed morphological analysis, clinical correlation, and discuss potential diagnoses with supporting evidence from the imaging findings.",
|
461 |
+
""
|
462 |
+
]
|
463 |
+
],
|
464 |
+
inputs=[image_input, clinical_question, patient_history],
|
465 |
+
label="Click any example to load it into the interface"
|
466 |
+
)
|
467 |
+
|
468 |
+
# Event handlers
|
469 |
+
analyze_btn.click(
|
470 |
+
fn=analyze_medical_image_llava,
|
471 |
+
inputs=[image_input, clinical_question, patient_history],
|
472 |
+
outputs=output,
|
473 |
+
show_progress=True
|
474 |
+
)
|
475 |
+
|
476 |
+
def clear_all_fields():
|
477 |
+
return None, "", "", ""
|
478 |
+
|
479 |
+
clear_btn.click(
|
480 |
+
fn=clear_all_fields,
|
481 |
+
outputs=[image_input, clinical_question, patient_history, output]
|
482 |
+
)
|
483 |
+
|
484 |
+
refresh_stats_btn.click(
|
485 |
+
fn=get_usage_stats,
|
486 |
+
outputs=stats_display
|
487 |
+
)
|
488 |
+
|
489 |
+
# Quick example button handlers
|
490 |
+
if llava_ready:
|
491 |
+
radiology_btn.click(
|
492 |
+
fn=lambda: ("Perform systematic radiological analysis of this medical image. Assess technical quality, identify normal anatomical structures, detect any pathological findings, and provide clinical interpretation with differential diagnosis.", "Adult patient with respiratory symptoms"),
|
493 |
+
outputs=[clinical_question, patient_history]
|
494 |
+
)
|
495 |
|
496 |
+
pathology_btn.click(
|
497 |
+
fn=lambda: ("Analyze this pathological specimen or medical image. Describe morphological features, identify cellular patterns, assess for pathological changes, and provide histopathological interpretation with clinical significance.", "Tissue sample for pathological evaluation"),
|
498 |
+
outputs=[clinical_question, patient_history]
|
499 |
+
)
|
500 |
+
|
501 |
+
dermatology_btn.click(
|
502 |
+
fn=lambda: ("Examine this dermatological image systematically. Describe the lesion characteristics, assess morphological features, evaluate for concerning signs, and provide differential diagnosis with clinical recommendations.", "Patient presenting with skin lesion requiring evaluation"),
|
503 |
+
outputs=[clinical_question, patient_history]
|
504 |
+
)
|
505 |
+
|
506 |
+
differential_btn.click(
|
507 |
+
fn=lambda: ("Based on the imaging findings in this medical image, provide a comprehensive differential diagnosis. List possible conditions in order of likelihood, explain supporting evidence for each diagnosis, and recommend additional studies if needed.", "Patient requiring diagnostic workup based on imaging findings"),
|
508 |
+
outputs=[clinical_question, patient_history]
|
509 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
510 |
|
511 |
+
# Comprehensive footer with detailed information
|
512 |
+
gr.Markdown("""
|
513 |
+
---
|
514 |
+
## π€ About LLaVA Medical AI
|
515 |
+
|
516 |
+
**LLaVA (Large Language and Vision Assistant)** is a state-of-the-art multimodal AI model that combines advanced computer vision with natural language processing for comprehensive medical image analysis.
|
517 |
+
|
518 |
+
### π¬ Key Capabilities
|
519 |
+
|
520 |
+
**Medical Image Analysis:**
|
521 |
+
- **Radiology**: X-rays, CT scans, MRI, ultrasound interpretation
|
522 |
+
- **Pathology**: Histological analysis, tissue examination, cellular morphology
|
523 |
+
- **Dermatology**: Skin lesion analysis, dermatological condition assessment
|
524 |
+
- **Ophthalmology**: Retinal imaging, ocular pathology evaluation
|
525 |
+
|
526 |
+
**Clinical Reasoning:**
|
527 |
+
- Systematic medical image interpretation
|
528 |
+
- Differential diagnosis generation
|
529 |
+
- Clinical correlation and significance assessment
|
530 |
+
- Educational medical content and explanations
|
531 |
+
|
532 |
+
### π₯ Medical Education Applications
|
533 |
+
|
534 |
+
- **Medical Student Training**: Interactive case-based learning
|
535 |
+
- **Resident Education**: Systematic approach to image interpretation
|
536 |
+
- **Continuing Medical Education**: Advanced diagnostic reasoning
|
537 |
+
- **Research Applications**: Medical imaging analysis and documentation
|
538 |
+
|
539 |
+
### π Privacy & Compliance
|
540 |
+
|
541 |
+
- **No Data Storage**: All images processed in real-time, not stored
|
542 |
+
- **Educational Purpose**: Designed specifically for medical education and training
|
543 |
+
- **Privacy Protection**: No patient identifiable information should be uploaded
|
544 |
+
- **Professional Standards**: Adheres to medical AI ethics and best practices
|
545 |
+
|
546 |
+
### β‘ Technical Specifications
|
547 |
+
|
548 |
+
- **Model**: LLaVA-v1.6-Mistral-7B (Latest version)
|
549 |
+
- **Context Window**: 32,000 tokens for comprehensive analysis
|
550 |
+
- **Processing**: Real-time inference with detailed medical reasoning
|
551 |
+
- **Accuracy**: Research-grade performance on medical imaging tasks
|
552 |
+
|
553 |
+
### π Support & Resources
|
554 |
+
|
555 |
+
For technical support, feature requests, or educational partnerships, please contact our support team.
|
556 |
+
|
557 |
+
---
|
558 |
+
**Powered by**: LLaVA (Large Language and Vision Assistant) | **License**: Apache 2.0 | **Purpose**: Medical Education & Research
|
559 |
+
""")
|
560 |
|
561 |
+
return demo
|
|
|
|
|
562 |
|
563 |
+
# Launch the application
|
564 |
if __name__ == "__main__":
|
565 |
+
demo = create_interface()
|
566 |
+
demo.launch(
|
567 |
+
server_name="0.0.0.0",
|
568 |
+
server_port=7860,
|
569 |
+
show_error=True
|
570 |
+
)
|