Update app.py
Browse files
app.py
CHANGED
@@ -1,16 +1,70 @@
|
|
1 |
-
# app.py - Medical AI
|
2 |
import gradio as gr
|
3 |
import torch
|
4 |
-
from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration
|
5 |
-
from PIL import Image
|
6 |
import logging
|
7 |
from collections import defaultdict, Counter
|
8 |
import time
|
|
|
9 |
|
10 |
# Configure logging
|
11 |
logging.basicConfig(level=logging.INFO)
|
12 |
logger = logging.getLogger(__name__)
|
13 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
14 |
# Usage tracking
|
15 |
class UsageTracker:
|
16 |
def __init__(self):
|
@@ -37,7 +91,7 @@ class UsageTracker:
|
|
37 |
|
38 |
# Rate limiting
|
39 |
class RateLimiter:
|
40 |
-
def __init__(self, max_requests_per_hour=
|
41 |
self.max_requests_per_hour = max_requests_per_hour
|
42 |
self.requests = defaultdict(list)
|
43 |
|
@@ -54,43 +108,66 @@ class RateLimiter:
|
|
54 |
usage_tracker = UsageTracker()
|
55 |
rate_limiter = RateLimiter()
|
56 |
|
57 |
-
# Model configuration
|
58 |
MODEL_ID = "llava-hf/llava-v1.6-mistral-7b-hf"
|
59 |
|
60 |
# Global variables
|
61 |
model = None
|
62 |
processor = None
|
63 |
|
64 |
-
def
|
65 |
-
"""Load LLaVA model
|
66 |
global model, processor
|
67 |
|
68 |
try:
|
69 |
logger.info(f"Loading LLaVA model: {MODEL_ID}")
|
70 |
|
71 |
-
#
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
-
|
77 |
-
|
78 |
-
|
79 |
-
|
80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
81 |
|
82 |
-
|
83 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
84 |
|
85 |
except Exception as e:
|
86 |
logger.error(f"β Error loading LLaVA: {str(e)}")
|
|
|
87 |
return False
|
88 |
|
89 |
# Load model at startup
|
90 |
-
llava_ready =
|
91 |
|
92 |
def analyze_medical_image_llava(image, clinical_question, patient_history=""):
|
93 |
-
"""Analyze medical image using LLaVA"""
|
94 |
start_time = time.time()
|
95 |
|
96 |
# Rate limiting
|
@@ -100,7 +177,20 @@ def analyze_medical_image_llava(image, clinical_question, patient_history=""):
|
|
100 |
|
101 |
if not llava_ready or model is None:
|
102 |
usage_tracker.log_analysis(False, time.time() - start_time)
|
103 |
-
return "β LLaVA
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
104 |
|
105 |
if image is None:
|
106 |
return "β οΈ Please upload a medical image first."
|
@@ -111,147 +201,126 @@ def analyze_medical_image_llava(image, clinical_question, patient_history=""):
|
|
111 |
try:
|
112 |
logger.info("Starting LLaVA medical analysis...")
|
113 |
|
114 |
-
# Prepare
|
115 |
-
medical_prompt = f"""You are
|
116 |
-
|
117 |
-
|
118 |
-
|
119 |
-
|
120 |
-
|
121 |
-
|
122 |
-
|
123 |
-
**
|
124 |
-
|
125 |
-
|
126 |
-
**Clinical
|
127 |
-
|
128 |
-
**
|
129 |
-
|
130 |
-
|
131 |
-
1. **IMAGE QUALITY ASSESSMENT**
|
132 |
-
- Technical adequacy of the image
|
133 |
-
- Any artifacts or limitations
|
134 |
-
- Overall diagnostic quality
|
135 |
-
|
136 |
-
2. **SYSTEMATIC OBSERVATION**
|
137 |
-
- Describe what you see in detail
|
138 |
-
- Identify anatomical structures visible
|
139 |
-
- Note any normal findings
|
140 |
-
|
141 |
-
3. **ABNORMAL FINDINGS**
|
142 |
-
- Identify any pathological changes
|
143 |
-
- Describe abnormalities in detail
|
144 |
-
- Note their location and characteristics
|
145 |
-
|
146 |
-
4. **CLINICAL SIGNIFICANCE**
|
147 |
-
- Explain the importance of findings
|
148 |
-
- Relate to potential diagnoses
|
149 |
-
- Discuss clinical implications
|
150 |
-
|
151 |
-
5. **DIFFERENTIAL DIAGNOSIS**
|
152 |
-
- List possible conditions
|
153 |
-
- Explain reasoning for each
|
154 |
-
- Prioritize based on imaging findings
|
155 |
-
|
156 |
-
6. **RECOMMENDATIONS**
|
157 |
-
- Suggest additional imaging if needed
|
158 |
-
- Recommend clinical correlation
|
159 |
-
- Advise on follow-up or further evaluation
|
160 |
-
|
161 |
-
Please be thorough, educational, and professional in your analysis. Always emphasize that this is for educational purposes and requires professional medical validation."""
|
162 |
-
|
163 |
-
# Prepare conversation for LLaVA
|
164 |
-
conversation = [
|
165 |
-
{
|
166 |
-
"role": "user",
|
167 |
-
"content": [
|
168 |
-
{"type": "text", "text": medical_prompt},
|
169 |
-
{"type": "image", "image": image}
|
170 |
-
]
|
171 |
-
}
|
172 |
-
]
|
173 |
|
174 |
-
#
|
175 |
-
|
176 |
-
|
177 |
-
|
178 |
-
|
179 |
-
|
180 |
-
|
181 |
-
|
182 |
-
|
183 |
-
|
184 |
-
|
185 |
-
|
186 |
-
|
187 |
-
|
188 |
-
|
189 |
-
|
190 |
-
do_sample=True,
|
191 |
-
temperature=0.2, # Lower temperature for more focused medical analysis
|
192 |
-
top_p=0.9,
|
193 |
-
repetition_penalty=1.1,
|
194 |
-
pad_token_id=processor.tokenizer.eos_token_id
|
195 |
-
)
|
196 |
|
197 |
-
#
|
198 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
199 |
|
200 |
# Clean up response
|
201 |
-
response = generated_text.strip()
|
202 |
|
203 |
-
# Format the response
|
204 |
-
formatted_response = f"""# π₯ **LLaVA Medical
|
205 |
|
206 |
## **Clinical Question:** {clinical_question}
|
207 |
{f"## **Patient History:** {patient_history}" if patient_history.strip() else ""}
|
208 |
|
209 |
---
|
210 |
|
211 |
-
## π **
|
212 |
|
213 |
{response}
|
214 |
|
215 |
---
|
216 |
|
217 |
-
## π **
|
|
|
|
|
218 |
|
219 |
**Key Points:**
|
220 |
-
-
|
221 |
-
-
|
222 |
-
-
|
223 |
-
|
224 |
-
**Clinical Workflow:**
|
225 |
-
1. **Review** the systematic analysis above
|
226 |
-
2. **Correlate** findings with patient symptoms and history
|
227 |
-
3. **Consult** with appropriate medical specialists as needed
|
228 |
-
4. **Document** findings in the patient's medical record
|
229 |
-
5. **Follow up** with recommended additional studies if indicated
|
230 |
-
|
231 |
-
**Educational Value:**
|
232 |
-
This analysis demonstrates structured medical image interpretation methodology and clinical reasoning processes used in healthcare settings.
|
233 |
"""
|
234 |
|
235 |
-
# Add
|
236 |
disclaimer = """
|
237 |
---
|
238 |
-
## β οΈ **
|
239 |
|
240 |
-
**FOR EDUCATIONAL
|
241 |
|
242 |
-
- **Not
|
243 |
-
- **Professional Review
|
244 |
-
- **Emergency
|
245 |
-
- **
|
246 |
-
- **
|
247 |
-
- **Educational Tool**: Designed for medical education, training, and research applications only
|
248 |
-
- **Data Privacy**: Do not upload images containing patient identifiable information
|
249 |
-
|
250 |
-
**Always consult qualified healthcare professionals for medical diagnosis and treatment decisions.**
|
251 |
|
252 |
---
|
253 |
-
**Powered by**: LLaVA (Large Language and Vision Assistant)
|
254 |
-
|
255 |
|
256 |
# Log successful analysis
|
257 |
duration = time.time() - start_time
|
@@ -266,58 +335,58 @@ This analysis demonstrates structured medical image interpretation methodology a
|
|
266 |
usage_tracker.log_analysis(False, duration)
|
267 |
logger.error(f"β LLaVA analysis error: {str(e)}")
|
268 |
|
269 |
-
|
270 |
-
|
271 |
-
|
272 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
273 |
|
274 |
def classify_question(question):
|
275 |
"""Classify clinical question type"""
|
276 |
question_lower = question.lower()
|
277 |
-
if any(word in question_lower for word in ['describe', 'findings', 'observe'
|
278 |
return 'descriptive'
|
279 |
-
elif any(word in question_lower for word in ['diagnosis', 'differential', 'condition'
|
280 |
return 'diagnostic'
|
281 |
-
elif any(word in question_lower for word in ['abnormal', 'pathology', '
|
282 |
return 'pathological'
|
283 |
-
elif any(word in question_lower for word in ['analyze', 'assess', 'evaluate', 'review']):
|
284 |
-
return 'analytical'
|
285 |
else:
|
286 |
return 'general'
|
287 |
|
288 |
def get_usage_stats():
|
289 |
-
"""Get
|
290 |
stats = usage_tracker.stats
|
291 |
if stats['total_analyses'] == 0:
|
292 |
return "π **Usage Statistics**\n\nNo analyses performed yet."
|
293 |
|
294 |
success_rate = (stats['successful_analyses'] / stats['total_analyses']) * 100
|
295 |
|
296 |
-
return f"""π **LLaVA
|
297 |
-
|
298 |
-
**Performance
|
299 |
-
-
|
300 |
-
-
|
301 |
-
-
|
302 |
-
|
303 |
-
|
304 |
-
|
305 |
-
|
306 |
-
|
307 |
-
**System Information:**
|
308 |
-
- **Model**: LLaVA-v1.6-Mistral-7B
|
309 |
-
- **Capabilities**: Medical image analysis and clinical reasoning
|
310 |
-
- **Device**: {'GPU' if torch.cuda.is_available() else 'CPU'}
|
311 |
-
- **Status**: {'π’ Operational' if llava_ready else 'π΄ Offline'}
|
312 |
"""
|
313 |
|
314 |
-
# Create
|
315 |
def create_interface():
|
316 |
with gr.Blocks(
|
317 |
-
title="LLaVA Medical
|
318 |
theme=gr.themes.Soft(),
|
319 |
css="""
|
320 |
-
.gradio-container { max-width:
|
321 |
.disclaimer { background-color: #fef2f2; border: 1px solid #fecaca; border-radius: 8px; padding: 16px; margin: 16px 0; }
|
322 |
.success { background-color: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin: 16px 0; }
|
323 |
.warning { background-color: #fffbeb; border: 1px solid #fed7aa; border-radius: 8px; padding: 16px; margin: 16px 0; }
|
@@ -330,8 +399,7 @@ def create_interface():
|
|
330 |
|
331 |
**Advanced Medical AI powered by LLaVA (Large Language and Vision Assistant)**
|
332 |
|
333 |
-
**
|
334 |
-
π« **Radiology** β’ π¬ **Pathology** β’ π©Ί **Dermatology** β’ ποΈ **Ophthalmology** β’ π§ **Clinical Reasoning**
|
335 |
""")
|
336 |
|
337 |
# Status display
|
@@ -339,130 +407,100 @@ def create_interface():
|
|
339 |
gr.Markdown("""
|
340 |
<div class="success">
|
341 |
β
<strong>LLAVA MEDICAL AI READY</strong><br>
|
342 |
-
LLaVA
|
343 |
</div>
|
344 |
""")
|
345 |
else:
|
346 |
gr.Markdown("""
|
347 |
<div class="warning">
|
348 |
-
β οΈ <strong>MODEL LOADING
|
349 |
-
LLaVA model
|
350 |
</div>
|
351 |
""")
|
352 |
|
353 |
# Medical disclaimer
|
354 |
gr.Markdown("""
|
355 |
<div class="disclaimer">
|
356 |
-
β οΈ <strong>
|
357 |
-
This AI
|
358 |
-
|
359 |
-
<strong>Do NOT upload real patient data or PHI.</strong> Always consult qualified healthcare professionals for medical decisions.
|
360 |
</div>
|
361 |
""")
|
362 |
|
363 |
with gr.Row():
|
364 |
-
# Left column
|
365 |
with gr.Column(scale=2):
|
366 |
with gr.Row():
|
367 |
with gr.Column():
|
368 |
-
gr.Markdown("## π€ Medical Image
|
369 |
image_input = gr.Image(
|
370 |
label="Upload Medical Image",
|
371 |
type="pil",
|
372 |
-
height=
|
373 |
-
sources=["upload", "clipboard"]
|
374 |
)
|
375 |
|
376 |
with gr.Column():
|
377 |
gr.Markdown("## π¬ Clinical Information")
|
378 |
clinical_question = gr.Textbox(
|
379 |
label="Clinical Question *",
|
380 |
-
placeholder="Examples:\n
|
381 |
-
lines=
|
382 |
-
max_lines=8
|
383 |
)
|
384 |
|
385 |
patient_history = gr.Textbox(
|
386 |
-
label="Patient History
|
387 |
-
placeholder="e.g.,
|
388 |
-
lines=
|
389 |
-
max_lines=5
|
390 |
)
|
391 |
|
392 |
with gr.Row():
|
393 |
-
clear_btn = gr.Button("ποΈ Clear
|
394 |
analyze_btn = gr.Button("π Analyze with LLaVA", variant="primary", size="lg")
|
395 |
|
396 |
-
gr.Markdown("## π
|
397 |
output = gr.Textbox(
|
398 |
-
label="
|
399 |
-
lines=
|
400 |
-
max_lines=50,
|
401 |
show_copy_button=True,
|
402 |
-
placeholder="Upload a medical image and
|
403 |
)
|
404 |
|
405 |
-
# Right column
|
406 |
with gr.Column(scale=1):
|
407 |
gr.Markdown("## βΉοΈ System Status")
|
408 |
|
409 |
-
|
410 |
-
device_info = "GPU" if torch.cuda.is_available() else "CPU"
|
411 |
|
412 |
gr.Markdown(f"""
|
413 |
-
**Model Status:** {
|
414 |
**AI Model:** LLaVA-v1.6-Mistral-7B
|
415 |
-
**Device:** {
|
416 |
-
**
|
417 |
-
**
|
418 |
-
**Rate Limit:** 30 requests/hour
|
419 |
""")
|
420 |
|
421 |
-
gr.Markdown("## π Usage
|
422 |
stats_display = gr.Markdown("")
|
423 |
-
refresh_stats_btn = gr.Button("π Refresh
|
424 |
|
425 |
if llava_ready:
|
426 |
-
gr.Markdown("## π― Quick
|
427 |
-
|
428 |
-
|
429 |
-
|
430 |
-
|
431 |
-
|
432 |
-
|
433 |
-
gr.Markdown("## π₯ Medical Specialties")
|
434 |
-
gr.Markdown("""
|
435 |
-
**LLaVA excels in:**
|
436 |
-
- Radiology interpretation
|
437 |
-
- Pathological analysis
|
438 |
-
- Dermatological assessment
|
439 |
-
- Ophthalmological evaluation
|
440 |
-
- Clinical reasoning & education
|
441 |
-
""")
|
442 |
-
|
443 |
-
# Comprehensive example cases
|
444 |
if llava_ready:
|
445 |
-
with gr.Accordion("π
|
446 |
examples = gr.Examples(
|
447 |
examples=[
|
448 |
[
|
449 |
"https://upload.wikimedia.org/wikipedia/commons/c/c8/Chest_Xray_PA_3-8-2010.png",
|
450 |
-
"Please
|
451 |
-
"Adult patient
|
452 |
-
],
|
453 |
-
[
|
454 |
-
None,
|
455 |
-
"Analyze this medical image systematically. Describe normal anatomical structures, identify any abnormal findings, assess clinical significance, and provide appropriate differential diagnoses based on imaging characteristics.",
|
456 |
-
"Patient with acute presentation requiring medical imaging evaluation"
|
457 |
-
],
|
458 |
-
[
|
459 |
-
None,
|
460 |
-
"What pathological changes are visible in this medical image? Please provide detailed morphological analysis, clinical correlation, and discuss potential diagnoses with supporting evidence from the imaging findings.",
|
461 |
-
""
|
462 |
]
|
463 |
],
|
464 |
-
inputs=[image_input, clinical_question, patient_history]
|
465 |
-
label="Click any example to load it into the interface"
|
466 |
)
|
467 |
|
468 |
# Event handlers
|
@@ -473,11 +511,8 @@ def create_interface():
|
|
473 |
show_progress=True
|
474 |
)
|
475 |
|
476 |
-
def clear_all_fields():
|
477 |
-
return None, "", "", ""
|
478 |
-
|
479 |
clear_btn.click(
|
480 |
-
fn=
|
481 |
outputs=[image_input, clinical_question, patient_history, output]
|
482 |
)
|
483 |
|
@@ -486,76 +521,37 @@ def create_interface():
|
|
486 |
outputs=stats_display
|
487 |
)
|
488 |
|
489 |
-
# Quick example
|
490 |
if llava_ready:
|
491 |
-
|
492 |
-
fn=lambda: ("
|
493 |
-
outputs=[clinical_question, patient_history]
|
494 |
-
)
|
495 |
-
|
496 |
-
pathology_btn.click(
|
497 |
-
fn=lambda: ("Analyze this pathological specimen or medical image. Describe morphological features, identify cellular patterns, assess for pathological changes, and provide histopathological interpretation with clinical significance.", "Tissue sample for pathological evaluation"),
|
498 |
outputs=[clinical_question, patient_history]
|
499 |
)
|
500 |
|
501 |
-
|
502 |
-
fn=lambda: ("
|
503 |
outputs=[clinical_question, patient_history]
|
504 |
)
|
505 |
|
506 |
-
|
507 |
-
fn=lambda: ("
|
508 |
outputs=[clinical_question, patient_history]
|
509 |
)
|
510 |
|
511 |
-
#
|
512 |
gr.Markdown("""
|
513 |
---
|
514 |
-
|
515 |
-
|
516 |
-
**LLaVA (Large Language and Vision Assistant)** is a state-of-the-art multimodal AI model that combines advanced computer vision with natural language processing for comprehensive medical image analysis.
|
517 |
-
|
518 |
-
### π¬ Key Capabilities
|
519 |
-
|
520 |
-
**Medical Image Analysis:**
|
521 |
-
- **Radiology**: X-rays, CT scans, MRI, ultrasound interpretation
|
522 |
-
- **Pathology**: Histological analysis, tissue examination, cellular morphology
|
523 |
-
- **Dermatology**: Skin lesion analysis, dermatological condition assessment
|
524 |
-
- **Ophthalmology**: Retinal imaging, ocular pathology evaluation
|
525 |
|
526 |
-
**
|
527 |
-
- Systematic medical image interpretation
|
528 |
-
- Differential diagnosis generation
|
529 |
-
- Clinical correlation and significance assessment
|
530 |
-
- Educational medical content and explanations
|
531 |
|
532 |
-
|
|
|
|
|
|
|
|
|
533 |
|
534 |
-
- **Medical
|
535 |
-
- **Resident Education**: Systematic approach to image interpretation
|
536 |
-
- **Continuing Medical Education**: Advanced diagnostic reasoning
|
537 |
-
- **Research Applications**: Medical imaging analysis and documentation
|
538 |
-
|
539 |
-
### π Privacy & Compliance
|
540 |
-
|
541 |
-
- **No Data Storage**: All images processed in real-time, not stored
|
542 |
-
- **Educational Purpose**: Designed specifically for medical education and training
|
543 |
-
- **Privacy Protection**: No patient identifiable information should be uploaded
|
544 |
-
- **Professional Standards**: Adheres to medical AI ethics and best practices
|
545 |
-
|
546 |
-
### β‘ Technical Specifications
|
547 |
-
|
548 |
-
- **Model**: LLaVA-v1.6-Mistral-7B (Latest version)
|
549 |
-
- **Context Window**: 32,000 tokens for comprehensive analysis
|
550 |
-
- **Processing**: Real-time inference with detailed medical reasoning
|
551 |
-
- **Accuracy**: Research-grade performance on medical imaging tasks
|
552 |
-
|
553 |
-
### π Support & Resources
|
554 |
-
|
555 |
-
For technical support, feature requests, or educational partnerships, please contact our support team.
|
556 |
-
|
557 |
-
---
|
558 |
-
**Powered by**: LLaVA (Large Language and Vision Assistant) | **License**: Apache 2.0 | **Purpose**: Medical Education & Research
|
559 |
""")
|
560 |
|
561 |
return demo
|
|
|
1 |
+
# app.py - Fixed LLaVA Medical AI with NoneType Error Resolution
|
2 |
import gradio as gr
|
3 |
import torch
|
|
|
|
|
4 |
import logging
|
5 |
from collections import defaultdict, Counter
|
6 |
import time
|
7 |
+
import traceback
|
8 |
|
9 |
# Configure logging
|
10 |
logging.basicConfig(level=logging.INFO)
|
11 |
logger = logging.getLogger(__name__)
|
12 |
|
13 |
+
# Fix the NoneType compatibility issue
|
14 |
+
def fix_transformers_compatibility():
|
15 |
+
"""Fix compatibility issues with transformers library"""
|
16 |
+
try:
|
17 |
+
# Import and fix the parallel styles issue
|
18 |
+
import transformers.modeling_utils as modeling_utils
|
19 |
+
if not hasattr(modeling_utils, 'ALL_PARALLEL_STYLES'):
|
20 |
+
modeling_utils.ALL_PARALLEL_STYLES = []
|
21 |
+
elif getattr(modeling_utils, 'ALL_PARALLEL_STYLES', None) is None:
|
22 |
+
modeling_utils.ALL_PARALLEL_STYLES = []
|
23 |
+
|
24 |
+
# Fix in specific model files
|
25 |
+
try:
|
26 |
+
import transformers.models.llava_next.modeling_llava_next as llava_next
|
27 |
+
if not hasattr(llava_next, 'ALL_PARALLEL_STYLES'):
|
28 |
+
llava_next.ALL_PARALLEL_STYLES = []
|
29 |
+
elif getattr(llava_next, 'ALL_PARALLEL_STYLES', None) is None:
|
30 |
+
llava_next.ALL_PARALLEL_STYLES = []
|
31 |
+
except ImportError:
|
32 |
+
pass
|
33 |
+
|
34 |
+
# Fix in mistral files if they exist
|
35 |
+
try:
|
36 |
+
import transformers.models.mistral.modeling_mistral as mistral
|
37 |
+
if not hasattr(mistral, 'ALL_PARALLEL_STYLES'):
|
38 |
+
mistral.ALL_PARALLEL_STYLES = []
|
39 |
+
elif getattr(mistral, 'ALL_PARALLEL_STYLES', None) is None:
|
40 |
+
mistral.ALL_PARALLEL_STYLES = []
|
41 |
+
except ImportError:
|
42 |
+
pass
|
43 |
+
|
44 |
+
logger.info("β
Applied compatibility fixes")
|
45 |
+
return True
|
46 |
+
except Exception as e:
|
47 |
+
logger.warning(f"β οΈ Could not apply compatibility fixes: {e}")
|
48 |
+
return False
|
49 |
+
|
50 |
+
# Apply compatibility fix before imports
|
51 |
+
fix_transformers_compatibility()
|
52 |
+
|
53 |
+
# Now import transformers
|
54 |
+
try:
|
55 |
+
from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration
|
56 |
+
from PIL import Image
|
57 |
+
logger.info("β
Transformers imported successfully")
|
58 |
+
except Exception as e:
|
59 |
+
logger.error(f"β Failed to import transformers: {e}")
|
60 |
+
# Fallback imports
|
61 |
+
try:
|
62 |
+
from transformers import LlavaProcessor, LlavaForConditionalGeneration as LlavaNextForConditionalGeneration
|
63 |
+
from transformers import AutoProcessor as LlavaNextProcessor
|
64 |
+
logger.info("β
Using fallback LLaVA imports")
|
65 |
+
except Exception as e2:
|
66 |
+
logger.error(f"β Fallback imports also failed: {e2}")
|
67 |
+
|
68 |
# Usage tracking
|
69 |
class UsageTracker:
|
70 |
def __init__(self):
|
|
|
91 |
|
92 |
# Rate limiting
|
93 |
class RateLimiter:
|
94 |
+
def __init__(self, max_requests_per_hour=20):
|
95 |
self.max_requests_per_hour = max_requests_per_hour
|
96 |
self.requests = defaultdict(list)
|
97 |
|
|
|
108 |
usage_tracker = UsageTracker()
|
109 |
rate_limiter = RateLimiter()
|
110 |
|
111 |
+
# Model configuration
|
112 |
MODEL_ID = "llava-hf/llava-v1.6-mistral-7b-hf"
|
113 |
|
114 |
# Global variables
|
115 |
model = None
|
116 |
processor = None
|
117 |
|
118 |
+
def load_llava_safe():
|
119 |
+
"""Load LLaVA model with comprehensive error handling"""
|
120 |
global model, processor
|
121 |
|
122 |
try:
|
123 |
logger.info(f"Loading LLaVA model: {MODEL_ID}")
|
124 |
|
125 |
+
# Try different loading approaches
|
126 |
+
loading_methods = [
|
127 |
+
("Standard LlavaNext", lambda: (
|
128 |
+
LlavaNextProcessor.from_pretrained(MODEL_ID),
|
129 |
+
LlavaNextForConditionalGeneration.from_pretrained(
|
130 |
+
MODEL_ID,
|
131 |
+
torch_dtype=torch.float32, # Use float32 for stability
|
132 |
+
device_map=None, # Let PyTorch handle device placement
|
133 |
+
low_cpu_mem_usage=True,
|
134 |
+
attn_implementation="eager" # Use eager attention to avoid issues
|
135 |
+
)
|
136 |
+
)),
|
137 |
+
("Auto Processor Fallback", lambda: (
|
138 |
+
LlavaNextProcessor.from_pretrained(MODEL_ID),
|
139 |
+
LlavaNextForConditionalGeneration.from_pretrained(
|
140 |
+
MODEL_ID,
|
141 |
+
torch_dtype=torch.float32,
|
142 |
+
trust_remote_code=True,
|
143 |
+
use_safetensors=True
|
144 |
+
)
|
145 |
+
)),
|
146 |
+
]
|
147 |
|
148 |
+
for method_name, method_func in loading_methods:
|
149 |
+
try:
|
150 |
+
logger.info(f"Trying {method_name}...")
|
151 |
+
processor, model = method_func()
|
152 |
+
logger.info(f"β
LLaVA loaded successfully using {method_name}!")
|
153 |
+
return True
|
154 |
+
except Exception as e:
|
155 |
+
logger.warning(f"β {method_name} failed: {str(e)}")
|
156 |
+
continue
|
157 |
+
|
158 |
+
logger.error("β All loading methods failed")
|
159 |
+
return False
|
160 |
|
161 |
except Exception as e:
|
162 |
logger.error(f"β Error loading LLaVA: {str(e)}")
|
163 |
+
logger.error(f"Full traceback: {traceback.format_exc()}")
|
164 |
return False
|
165 |
|
166 |
# Load model at startup
|
167 |
+
llava_ready = load_llava_safe()
|
168 |
|
169 |
def analyze_medical_image_llava(image, clinical_question, patient_history=""):
|
170 |
+
"""Analyze medical image using LLaVA with robust error handling"""
|
171 |
start_time = time.time()
|
172 |
|
173 |
# Rate limiting
|
|
|
177 |
|
178 |
if not llava_ready or model is None:
|
179 |
usage_tracker.log_analysis(False, time.time() - start_time)
|
180 |
+
return """β **LLaVA Model Loading Issue**
|
181 |
+
|
182 |
+
The LLaVA model failed to load due to compatibility issues. This is often caused by:
|
183 |
+
|
184 |
+
1. **Library Version Conflicts**: Try refreshing the page - we've applied compatibility fixes
|
185 |
+
2. **Memory Constraints**: The 7B model requires significant resources
|
186 |
+
3. **Transformers Version**: Some versions have compatibility issues
|
187 |
+
|
188 |
+
**Suggested Solutions:**
|
189 |
+
- **Refresh the page** and wait 2-3 minutes for model loading
|
190 |
+
- **Upgrade to GPU hardware** for better performance and stability
|
191 |
+
- **Try a different image** if the issue persists
|
192 |
+
|
193 |
+
**Technical Info**: There may be version conflicts in the transformers library. The model files downloaded successfully but initialization failed."""
|
194 |
|
195 |
if image is None:
|
196 |
return "β οΈ Please upload a medical image first."
|
|
|
201 |
try:
|
202 |
logger.info("Starting LLaVA medical analysis...")
|
203 |
|
204 |
+
# Prepare medical prompt
|
205 |
+
medical_prompt = f"""You are an expert medical AI assistant analyzing medical images. Please provide a comprehensive medical analysis.
|
206 |
+
|
207 |
+
{f"Patient History: {patient_history}" if patient_history.strip() else ""}
|
208 |
+
|
209 |
+
Clinical Question: {clinical_question}
|
210 |
+
|
211 |
+
Please analyze this medical image systematically:
|
212 |
+
|
213 |
+
1. **Image Quality**: Assess technical quality and diagnostic adequacy
|
214 |
+
2. **Anatomical Structures**: Identify visible normal structures
|
215 |
+
3. **Abnormal Findings**: Describe any pathological changes
|
216 |
+
4. **Clinical Significance**: Explain the importance of findings
|
217 |
+
5. **Assessment**: Provide clinical interpretation
|
218 |
+
6. **Recommendations**: Suggest next steps if appropriate
|
219 |
+
|
220 |
+
Provide detailed, educational medical analysis suitable for learning purposes."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
221 |
|
222 |
+
# Different prompt formats to try
|
223 |
+
prompt_formats = [
|
224 |
+
# Format 1: Simple user message
|
225 |
+
lambda: f"USER: <image>\n{medical_prompt}\nASSISTANT:",
|
226 |
+
|
227 |
+
# Format 2: Chat format
|
228 |
+
lambda: processor.apply_chat_template([
|
229 |
+
{"role": "user", "content": [
|
230 |
+
{"type": "image", "image": image},
|
231 |
+
{"type": "text", "text": medical_prompt}
|
232 |
+
]}
|
233 |
+
], add_generation_prompt=True),
|
234 |
+
|
235 |
+
# Format 3: Direct format
|
236 |
+
lambda: medical_prompt
|
237 |
+
]
|
|
|
|
|
|
|
|
|
|
|
|
|
238 |
|
239 |
+
# Try different prompt formats
|
240 |
+
for i, prompt_func in enumerate(prompt_formats):
|
241 |
+
try:
|
242 |
+
logger.info(f"Trying prompt format {i+1}...")
|
243 |
+
|
244 |
+
if i == 1: # Chat template format
|
245 |
+
try:
|
246 |
+
prompt = prompt_func()
|
247 |
+
except:
|
248 |
+
continue
|
249 |
+
else:
|
250 |
+
prompt = prompt_func()
|
251 |
+
|
252 |
+
# Process inputs
|
253 |
+
inputs = processor(prompt, image, return_tensors='pt')
|
254 |
+
|
255 |
+
# Generate response with conservative settings
|
256 |
+
logger.info("Generating medical analysis...")
|
257 |
+
with torch.inference_mode():
|
258 |
+
output = model.generate(
|
259 |
+
**inputs,
|
260 |
+
max_new_tokens=1000, # Conservative limit
|
261 |
+
do_sample=True,
|
262 |
+
temperature=0.3,
|
263 |
+
top_p=0.9,
|
264 |
+
repetition_penalty=1.1,
|
265 |
+
use_cache=False # Disable cache for stability
|
266 |
+
)
|
267 |
+
|
268 |
+
# Decode response
|
269 |
+
generated_text = processor.decode(output[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True)
|
270 |
+
|
271 |
+
if generated_text and generated_text.strip():
|
272 |
+
break
|
273 |
+
|
274 |
+
except Exception as e:
|
275 |
+
logger.warning(f"Prompt format {i+1} failed: {e}")
|
276 |
+
if i == len(prompt_formats) - 1: # Last attempt
|
277 |
+
raise e
|
278 |
+
continue
|
279 |
|
280 |
# Clean up response
|
281 |
+
response = generated_text.strip() if generated_text else "Analysis completed."
|
282 |
|
283 |
+
# Format the response
|
284 |
+
formatted_response = f"""# π₯ **LLaVA Medical Analysis**
|
285 |
|
286 |
## **Clinical Question:** {clinical_question}
|
287 |
{f"## **Patient History:** {patient_history}" if patient_history.strip() else ""}
|
288 |
|
289 |
---
|
290 |
|
291 |
+
## π **Medical Analysis Results**
|
292 |
|
293 |
{response}
|
294 |
|
295 |
---
|
296 |
|
297 |
+
## π **Clinical Summary**
|
298 |
+
|
299 |
+
This analysis was generated using LLaVA (Large Language and Vision Assistant) for educational purposes. The findings should be interpreted by qualified medical professionals and correlated with clinical presentation.
|
300 |
|
301 |
**Key Points:**
|
302 |
+
- Analysis based on visual medical image interpretation
|
303 |
+
- Systematic approach to medical imaging assessment
|
304 |
+
- Educational tool for medical learning and training
|
305 |
+
- Requires professional medical validation
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
306 |
"""
|
307 |
|
308 |
+
# Add medical disclaimer
|
309 |
disclaimer = """
|
310 |
---
|
311 |
+
## β οΈ **MEDICAL DISCLAIMER**
|
312 |
|
313 |
+
**FOR EDUCATIONAL PURPOSES ONLY**
|
314 |
|
315 |
+
- **Not Diagnostic**: This AI analysis is not a medical diagnosis
|
316 |
+
- **Professional Review**: All findings require validation by healthcare professionals
|
317 |
+
- **Emergency Care**: Contact emergency services for urgent medical concerns
|
318 |
+
- **Educational Tool**: Designed for medical education and training
|
319 |
+
- **No PHI**: Do not upload patient identifiable information
|
|
|
|
|
|
|
|
|
320 |
|
321 |
---
|
322 |
+
**Powered by**: LLaVA (Large Language and Vision Assistant)
|
323 |
+
"""
|
324 |
|
325 |
# Log successful analysis
|
326 |
duration = time.time() - start_time
|
|
|
335 |
usage_tracker.log_analysis(False, duration)
|
336 |
logger.error(f"β LLaVA analysis error: {str(e)}")
|
337 |
|
338 |
+
return f"""β **Analysis Error**
|
339 |
+
|
340 |
+
The analysis failed with error: {str(e)}
|
341 |
+
|
342 |
+
**Common Solutions:**
|
343 |
+
- **Try again**: Sometimes temporary processing issues occur
|
344 |
+
- **Smaller image**: Try with a smaller or different format image
|
345 |
+
- **Simpler question**: Use a more straightforward clinical question
|
346 |
+
- **Refresh page**: Reload the page if model seems unstable
|
347 |
+
|
348 |
+
**Technical Details:** {str(e)[:200]}"""
|
349 |
|
350 |
def classify_question(question):
|
351 |
"""Classify clinical question type"""
|
352 |
question_lower = question.lower()
|
353 |
+
if any(word in question_lower for word in ['describe', 'findings', 'observe']):
|
354 |
return 'descriptive'
|
355 |
+
elif any(word in question_lower for word in ['diagnosis', 'differential', 'condition']):
|
356 |
return 'diagnostic'
|
357 |
+
elif any(word in question_lower for word in ['abnormal', 'pathology', 'disease']):
|
358 |
return 'pathological'
|
|
|
|
|
359 |
else:
|
360 |
return 'general'
|
361 |
|
362 |
def get_usage_stats():
|
363 |
+
"""Get usage statistics"""
|
364 |
stats = usage_tracker.stats
|
365 |
if stats['total_analyses'] == 0:
|
366 |
return "π **Usage Statistics**\n\nNo analyses performed yet."
|
367 |
|
368 |
success_rate = (stats['successful_analyses'] / stats['total_analyses']) * 100
|
369 |
|
370 |
+
return f"""π **LLaVA Usage Statistics**
|
371 |
+
|
372 |
+
**Performance:**
|
373 |
+
- Total Analyses: {stats['total_analyses']}
|
374 |
+
- Success Rate: {success_rate:.1f}%
|
375 |
+
- Avg Processing Time: {stats['average_processing_time']:.2f}s
|
376 |
+
|
377 |
+
**Popular Question Types:**
|
378 |
+
{chr(10).join([f"- {qtype}: {count}" for qtype, count in stats['question_types'].most_common(3)])}
|
379 |
+
|
380 |
+
**Model Status**: {'π’ Ready' if llava_ready else 'π΄ Loading Issues'}
|
|
|
|
|
|
|
|
|
|
|
381 |
"""
|
382 |
|
383 |
+
# Create Gradio interface
|
384 |
def create_interface():
|
385 |
with gr.Blocks(
|
386 |
+
title="LLaVA Medical Analysis",
|
387 |
theme=gr.themes.Soft(),
|
388 |
css="""
|
389 |
+
.gradio-container { max-width: 1200px !important; }
|
390 |
.disclaimer { background-color: #fef2f2; border: 1px solid #fecaca; border-radius: 8px; padding: 16px; margin: 16px 0; }
|
391 |
.success { background-color: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin: 16px 0; }
|
392 |
.warning { background-color: #fffbeb; border: 1px solid #fed7aa; border-radius: 8px; padding: 16px; margin: 16px 0; }
|
|
|
399 |
|
400 |
**Advanced Medical AI powered by LLaVA (Large Language and Vision Assistant)**
|
401 |
|
402 |
+
**Medical Capabilities:** π« Radiology β’ π¬ Pathology β’ π©Ί Dermatology β’ ποΈ Ophthalmology
|
|
|
403 |
""")
|
404 |
|
405 |
# Status display
|
|
|
407 |
gr.Markdown("""
|
408 |
<div class="success">
|
409 |
β
<strong>LLAVA MEDICAL AI READY</strong><br>
|
410 |
+
LLaVA model loaded successfully with compatibility fixes. Ready for medical image analysis.
|
411 |
</div>
|
412 |
""")
|
413 |
else:
|
414 |
gr.Markdown("""
|
415 |
<div class="warning">
|
416 |
+
β οΈ <strong>MODEL LOADING ISSUE</strong><br>
|
417 |
+
LLaVA model had loading problems. Try refreshing the page or contact support for assistance.
|
418 |
</div>
|
419 |
""")
|
420 |
|
421 |
# Medical disclaimer
|
422 |
gr.Markdown("""
|
423 |
<div class="disclaimer">
|
424 |
+
β οΈ <strong>MEDICAL DISCLAIMER</strong><br>
|
425 |
+
This AI provides medical analysis for <strong>educational purposes only</strong>.
|
426 |
+
Do not upload real patient data. Always consult healthcare professionals for medical decisions.
|
|
|
427 |
</div>
|
428 |
""")
|
429 |
|
430 |
with gr.Row():
|
431 |
+
# Left column
|
432 |
with gr.Column(scale=2):
|
433 |
with gr.Row():
|
434 |
with gr.Column():
|
435 |
+
gr.Markdown("## π€ Medical Image")
|
436 |
image_input = gr.Image(
|
437 |
label="Upload Medical Image",
|
438 |
type="pil",
|
439 |
+
height=300
|
|
|
440 |
)
|
441 |
|
442 |
with gr.Column():
|
443 |
gr.Markdown("## π¬ Clinical Information")
|
444 |
clinical_question = gr.Textbox(
|
445 |
label="Clinical Question *",
|
446 |
+
placeholder="Examples:\nοΏ½οΏ½ Analyze this medical image\nβ’ What abnormalities are visible?\nβ’ Describe the findings\nβ’ Provide medical interpretation",
|
447 |
+
lines=4
|
|
|
448 |
)
|
449 |
|
450 |
patient_history = gr.Textbox(
|
451 |
+
label="Patient History (Optional)",
|
452 |
+
placeholder="e.g., 45-year-old with chest pain",
|
453 |
+
lines=2
|
|
|
454 |
)
|
455 |
|
456 |
with gr.Row():
|
457 |
+
clear_btn = gr.Button("ποΈ Clear", variant="secondary")
|
458 |
analyze_btn = gr.Button("π Analyze with LLaVA", variant="primary", size="lg")
|
459 |
|
460 |
+
gr.Markdown("## π Medical Analysis Results")
|
461 |
output = gr.Textbox(
|
462 |
+
label="LLaVA Medical Analysis",
|
463 |
+
lines=20,
|
|
|
464 |
show_copy_button=True,
|
465 |
+
placeholder="Upload a medical image and clinical question..." if llava_ready else "Model loading issues - please refresh the page"
|
466 |
)
|
467 |
|
468 |
+
# Right column
|
469 |
with gr.Column(scale=1):
|
470 |
gr.Markdown("## βΉοΈ System Status")
|
471 |
|
472 |
+
status = "β
Ready" if llava_ready else "β οΈ Loading Issues"
|
|
|
473 |
|
474 |
gr.Markdown(f"""
|
475 |
+
**Model Status:** {status}
|
476 |
**AI Model:** LLaVA-v1.6-Mistral-7B
|
477 |
+
**Device:** {'GPU' if torch.cuda.is_available() else 'CPU'}
|
478 |
+
**Compatibility:** Fixed for stability
|
479 |
+
**Rate Limit:** 20 requests/hour
|
|
|
480 |
""")
|
481 |
|
482 |
+
gr.Markdown("## π Usage Statistics")
|
483 |
stats_display = gr.Markdown("")
|
484 |
+
refresh_stats_btn = gr.Button("π Refresh Stats", size="sm")
|
485 |
|
486 |
if llava_ready:
|
487 |
+
gr.Markdown("## π― Quick Examples")
|
488 |
+
general_btn = gr.Button("General Analysis", size="sm")
|
489 |
+
findings_btn = gr.Button("Find Abnormalities", size="sm")
|
490 |
+
interpret_btn = gr.Button("Medical Interpretation", size="sm")
|
491 |
+
|
492 |
+
# Example cases
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
493 |
if llava_ready:
|
494 |
+
with gr.Accordion("π Example Cases", open=False):
|
495 |
examples = gr.Examples(
|
496 |
examples=[
|
497 |
[
|
498 |
"https://upload.wikimedia.org/wikipedia/commons/c/c8/Chest_Xray_PA_3-8-2010.png",
|
499 |
+
"Please analyze this chest X-ray and describe any findings. Assess the image quality, identify normal structures, and note any abnormalities.",
|
500 |
+
"Adult patient with respiratory symptoms"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
501 |
]
|
502 |
],
|
503 |
+
inputs=[image_input, clinical_question, patient_history]
|
|
|
504 |
)
|
505 |
|
506 |
# Event handlers
|
|
|
511 |
show_progress=True
|
512 |
)
|
513 |
|
|
|
|
|
|
|
514 |
clear_btn.click(
|
515 |
+
fn=lambda: (None, "", "", ""),
|
516 |
outputs=[image_input, clinical_question, patient_history, output]
|
517 |
)
|
518 |
|
|
|
521 |
outputs=stats_display
|
522 |
)
|
523 |
|
524 |
+
# Quick example handlers
|
525 |
if llava_ready:
|
526 |
+
general_btn.click(
|
527 |
+
fn=lambda: ("Analyze this medical image comprehensively. Describe what you observe and provide medical interpretation.", ""),
|
|
|
|
|
|
|
|
|
|
|
528 |
outputs=[clinical_question, patient_history]
|
529 |
)
|
530 |
|
531 |
+
findings_btn.click(
|
532 |
+
fn=lambda: ("What abnormalities or pathological findings are visible in this medical image?", ""),
|
533 |
outputs=[clinical_question, patient_history]
|
534 |
)
|
535 |
|
536 |
+
interpret_btn.click(
|
537 |
+
fn=lambda: ("Provide medical interpretation of this image including clinical significance of any findings.", ""),
|
538 |
outputs=[clinical_question, patient_history]
|
539 |
)
|
540 |
|
541 |
+
# Footer
|
542 |
gr.Markdown("""
|
543 |
---
|
544 |
+
### π€ LLaVA Medical AI
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
545 |
|
546 |
+
**Large Language and Vision Assistant** optimized for medical image analysis with compatibility fixes for stable operation.
|
|
|
|
|
|
|
|
|
547 |
|
548 |
+
**Features:**
|
549 |
+
- Advanced medical image interpretation
|
550 |
+
- Systematic clinical analysis approach
|
551 |
+
- Educational medical explanations
|
552 |
+
- Comprehensive error handling
|
553 |
|
554 |
+
**Model:** LLaVA-v1.6-Mistral-7B | **Purpose:** Medical Education & Research
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
555 |
""")
|
556 |
|
557 |
return demo
|