walaa2022 commited on
Commit
0c3e999
Β·
verified Β·
1 Parent(s): c4470a3

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +290 -294
app.py CHANGED
@@ -1,16 +1,70 @@
1
- # app.py - Medical AI using LLaVA (Large Language and Vision Assistant)
2
  import gradio as gr
3
  import torch
4
- from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration
5
- from PIL import Image
6
  import logging
7
  from collections import defaultdict, Counter
8
  import time
 
9
 
10
  # Configure logging
11
  logging.basicConfig(level=logging.INFO)
12
  logger = logging.getLogger(__name__)
13
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  # Usage tracking
15
  class UsageTracker:
16
  def __init__(self):
@@ -37,7 +91,7 @@ class UsageTracker:
37
 
38
  # Rate limiting
39
  class RateLimiter:
40
- def __init__(self, max_requests_per_hour=30):
41
  self.max_requests_per_hour = max_requests_per_hour
42
  self.requests = defaultdict(list)
43
 
@@ -54,43 +108,66 @@ class RateLimiter:
54
  usage_tracker = UsageTracker()
55
  rate_limiter = RateLimiter()
56
 
57
- # Model configuration - Using LLaVA-Next (latest version)
58
  MODEL_ID = "llava-hf/llava-v1.6-mistral-7b-hf"
59
 
60
  # Global variables
61
  model = None
62
  processor = None
63
 
64
- def load_llava():
65
- """Load LLaVA model for medical analysis"""
66
  global model, processor
67
 
68
  try:
69
  logger.info(f"Loading LLaVA model: {MODEL_ID}")
70
 
71
- # Load processor
72
- processor = LlavaNextProcessor.from_pretrained(MODEL_ID)
73
-
74
- # Load model with appropriate settings
75
- model = LlavaNextForConditionalGeneration.from_pretrained(
76
- MODEL_ID,
77
- torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
78
- device_map="auto" if torch.cuda.is_available() else None,
79
- low_cpu_mem_usage=True
80
- )
 
 
 
 
 
 
 
 
 
 
 
 
81
 
82
- logger.info("βœ… LLaVA model loaded successfully!")
83
- return True
 
 
 
 
 
 
 
 
 
 
84
 
85
  except Exception as e:
86
  logger.error(f"❌ Error loading LLaVA: {str(e)}")
 
87
  return False
88
 
89
  # Load model at startup
90
- llava_ready = load_llava()
91
 
92
  def analyze_medical_image_llava(image, clinical_question, patient_history=""):
93
- """Analyze medical image using LLaVA"""
94
  start_time = time.time()
95
 
96
  # Rate limiting
@@ -100,7 +177,20 @@ def analyze_medical_image_llava(image, clinical_question, patient_history=""):
100
 
101
  if not llava_ready or model is None:
102
  usage_tracker.log_analysis(False, time.time() - start_time)
103
- return "❌ LLaVA model not loaded. Please refresh the page and wait for model loading."
 
 
 
 
 
 
 
 
 
 
 
 
 
104
 
105
  if image is None:
106
  return "⚠️ Please upload a medical image first."
@@ -111,147 +201,126 @@ def analyze_medical_image_llava(image, clinical_question, patient_history=""):
111
  try:
112
  logger.info("Starting LLaVA medical analysis...")
113
 
114
- # Prepare comprehensive medical prompt
115
- medical_prompt = f"""You are a highly skilled medical AI assistant with expertise in medical image analysis. You have extensive knowledge in:
116
-
117
- - **Radiology**: X-rays, CT scans, MRI, ultrasound interpretation
118
- - **Pathology**: Histological analysis, tissue examination, cellular patterns
119
- - **Dermatology**: Skin lesions, rashes, dermatological conditions
120
- - **Ophthalmology**: Retinal imaging, eye examinations, ocular pathology
121
- - **General Medical Imaging**: Cross-sectional anatomy, normal variants, pathological findings
122
-
123
- **Patient Information:**
124
- {f"Patient History: {patient_history}" if patient_history.strip() else "No specific patient history provided"}
125
-
126
- **Clinical Question:** {clinical_question}
127
-
128
- **Instructions:**
129
- Please provide a comprehensive medical analysis of this image following this structure:
130
-
131
- 1. **IMAGE QUALITY ASSESSMENT**
132
- - Technical adequacy of the image
133
- - Any artifacts or limitations
134
- - Overall diagnostic quality
135
-
136
- 2. **SYSTEMATIC OBSERVATION**
137
- - Describe what you see in detail
138
- - Identify anatomical structures visible
139
- - Note any normal findings
140
-
141
- 3. **ABNORMAL FINDINGS**
142
- - Identify any pathological changes
143
- - Describe abnormalities in detail
144
- - Note their location and characteristics
145
-
146
- 4. **CLINICAL SIGNIFICANCE**
147
- - Explain the importance of findings
148
- - Relate to potential diagnoses
149
- - Discuss clinical implications
150
-
151
- 5. **DIFFERENTIAL DIAGNOSIS**
152
- - List possible conditions
153
- - Explain reasoning for each
154
- - Prioritize based on imaging findings
155
-
156
- 6. **RECOMMENDATIONS**
157
- - Suggest additional imaging if needed
158
- - Recommend clinical correlation
159
- - Advise on follow-up or further evaluation
160
-
161
- Please be thorough, educational, and professional in your analysis. Always emphasize that this is for educational purposes and requires professional medical validation."""
162
-
163
- # Prepare conversation for LLaVA
164
- conversation = [
165
- {
166
- "role": "user",
167
- "content": [
168
- {"type": "text", "text": medical_prompt},
169
- {"type": "image", "image": image}
170
- ]
171
- }
172
- ]
173
 
174
- # Apply chat template
175
- prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
176
-
177
- # Process inputs
178
- inputs = processor(prompt, image, return_tensors='pt')
179
-
180
- # Move to appropriate device
181
- if torch.cuda.is_available() and hasattr(model, 'device'):
182
- inputs = {k: v.to(model.device) for k, v in inputs.items()}
183
-
184
- # Generate response
185
- logger.info("Generating comprehensive medical analysis...")
186
- with torch.inference_mode():
187
- output = model.generate(
188
- **inputs,
189
- max_new_tokens=2000,
190
- do_sample=True,
191
- temperature=0.2, # Lower temperature for more focused medical analysis
192
- top_p=0.9,
193
- repetition_penalty=1.1,
194
- pad_token_id=processor.tokenizer.eos_token_id
195
- )
196
 
197
- # Decode response
198
- generated_text = processor.decode(output[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
199
 
200
  # Clean up response
201
- response = generated_text.strip()
202
 
203
- # Format the response with medical structure
204
- formatted_response = f"""# πŸ₯ **LLaVA Medical Image Analysis**
205
 
206
  ## **Clinical Question:** {clinical_question}
207
  {f"## **Patient History:** {patient_history}" if patient_history.strip() else ""}
208
 
209
  ---
210
 
211
- ## πŸ” **Comprehensive Medical Analysis**
212
 
213
  {response}
214
 
215
  ---
216
 
217
- ## πŸ“‹ **Summary and Clinical Correlation**
 
 
218
 
219
  **Key Points:**
220
- - This analysis provides a systematic approach to medical image interpretation
221
- - All findings should be correlated with clinical presentation and patient history
222
- - The AI assessment serves as an educational tool and decision support aid
223
-
224
- **Clinical Workflow:**
225
- 1. **Review** the systematic analysis above
226
- 2. **Correlate** findings with patient symptoms and history
227
- 3. **Consult** with appropriate medical specialists as needed
228
- 4. **Document** findings in the patient's medical record
229
- 5. **Follow up** with recommended additional studies if indicated
230
-
231
- **Educational Value:**
232
- This analysis demonstrates structured medical image interpretation methodology and clinical reasoning processes used in healthcare settings.
233
  """
234
 
235
- # Add comprehensive medical disclaimer
236
  disclaimer = """
237
  ---
238
- ## ⚠️ **IMPORTANT MEDICAL DISCLAIMER**
239
 
240
- **FOR EDUCATIONAL AND RESEARCH PURPOSES ONLY**
241
 
242
- - **Not a Medical Diagnosis**: This AI analysis does not constitute a medical diagnosis, treatment recommendation, or professional medical advice
243
- - **Professional Review Required**: All findings must be validated by qualified healthcare professionals
244
- - **Emergency Situations**: For urgent medical concerns, contact emergency services immediately (911 in US)
245
- - **Clinical Correlation**: AI findings must be correlated with clinical examination and patient history
246
- - **Liability**: This system is not intended for clinical decision-making and users assume all responsibility
247
- - **Educational Tool**: Designed for medical education, training, and research applications only
248
- - **Data Privacy**: Do not upload images containing patient identifiable information
249
-
250
- **Always consult qualified healthcare professionals for medical diagnosis and treatment decisions.**
251
 
252
  ---
253
- **Powered by**: LLaVA (Large Language and Vision Assistant) | **Model**: {MODEL_ID}
254
- """
255
 
256
  # Log successful analysis
257
  duration = time.time() - start_time
@@ -266,58 +335,58 @@ This analysis demonstrates structured medical image interpretation methodology a
266
  usage_tracker.log_analysis(False, duration)
267
  logger.error(f"❌ LLaVA analysis error: {str(e)}")
268
 
269
- if "memory" in str(e).lower() or "cuda" in str(e).lower():
270
- return "❌ **Memory Error**: The model requires more memory. Try using a smaller image or upgrading to GPU hardware."
271
- else:
272
- return f"❌ **Analysis Failed**: {str(e)}\n\nPlease try again with a different image or contact support if the issue persists."
 
 
 
 
 
 
 
273
 
274
  def classify_question(question):
275
  """Classify clinical question type"""
276
  question_lower = question.lower()
277
- if any(word in question_lower for word in ['describe', 'findings', 'observe', 'see']):
278
  return 'descriptive'
279
- elif any(word in question_lower for word in ['diagnosis', 'differential', 'condition', 'disease']):
280
  return 'diagnostic'
281
- elif any(word in question_lower for word in ['abnormal', 'pathology', 'lesion', 'mass']):
282
  return 'pathological'
283
- elif any(word in question_lower for word in ['analyze', 'assess', 'evaluate', 'review']):
284
- return 'analytical'
285
  else:
286
  return 'general'
287
 
288
  def get_usage_stats():
289
- """Get comprehensive usage statistics"""
290
  stats = usage_tracker.stats
291
  if stats['total_analyses'] == 0:
292
  return "πŸ“Š **Usage Statistics**\n\nNo analyses performed yet."
293
 
294
  success_rate = (stats['successful_analyses'] / stats['total_analyses']) * 100
295
 
296
- return f"""πŸ“Š **LLaVA Medical AI Usage Statistics**
297
-
298
- **Performance Metrics:**
299
- - **Total Analyses**: {stats['total_analyses']}
300
- - **Success Rate**: {success_rate:.1f}%
301
- - **Average Processing Time**: {stats['average_processing_time']:.2f} seconds
302
- - **Failed Analyses**: {stats['failed_analyses']}
303
-
304
- **Question Type Distribution:**
305
- {chr(10).join([f"- **{qtype.title()}**: {count} ({count/stats['total_analyses']*100:.1f}%)" for qtype, count in stats['question_types'].most_common()])}
306
-
307
- **System Information:**
308
- - **Model**: LLaVA-v1.6-Mistral-7B
309
- - **Capabilities**: Medical image analysis and clinical reasoning
310
- - **Device**: {'GPU' if torch.cuda.is_available() else 'CPU'}
311
- - **Status**: {'🟒 Operational' if llava_ready else 'πŸ”΄ Offline'}
312
  """
313
 
314
- # Create comprehensive Gradio interface
315
  def create_interface():
316
  with gr.Blocks(
317
- title="LLaVA Medical Image Analysis",
318
  theme=gr.themes.Soft(),
319
  css="""
320
- .gradio-container { max-width: 1400px !important; }
321
  .disclaimer { background-color: #fef2f2; border: 1px solid #fecaca; border-radius: 8px; padding: 16px; margin: 16px 0; }
322
  .success { background-color: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin: 16px 0; }
323
  .warning { background-color: #fffbeb; border: 1px solid #fed7aa; border-radius: 8px; padding: 16px; margin: 16px 0; }
@@ -330,8 +399,7 @@ def create_interface():
330
 
331
  **Advanced Medical AI powered by LLaVA (Large Language and Vision Assistant)**
332
 
333
- **Specialized Medical Capabilities:**
334
- 🫁 **Radiology** β€’ πŸ”¬ **Pathology** β€’ 🩺 **Dermatology** β€’ πŸ‘οΈ **Ophthalmology** β€’ 🧠 **Clinical Reasoning**
335
  """)
336
 
337
  # Status display
@@ -339,130 +407,100 @@ def create_interface():
339
  gr.Markdown("""
340
  <div class="success">
341
  βœ… <strong>LLAVA MEDICAL AI READY</strong><br>
342
- LLaVA vision-language model loaded successfully. Ready for comprehensive medical image analysis with clinical reasoning.
343
  </div>
344
  """)
345
  else:
346
  gr.Markdown("""
347
  <div class="warning">
348
- ⚠️ <strong>MODEL LOADING IN PROGRESS</strong><br>
349
- LLaVA model is loading. This may take a few minutes. Please wait and refresh the page.
350
  </div>
351
  """)
352
 
353
  # Medical disclaimer
354
  gr.Markdown("""
355
  <div class="disclaimer">
356
- ⚠️ <strong>CRITICAL MEDICAL DISCLAIMER</strong><br>
357
- This AI tool provides <strong>educational medical analysis only</strong>. It is NOT a substitute for professional medical diagnosis.
358
- <br><br>
359
- <strong>Do NOT upload real patient data or PHI.</strong> Always consult qualified healthcare professionals for medical decisions.
360
  </div>
361
  """)
362
 
363
  with gr.Row():
364
- # Left column - Main interface
365
  with gr.Column(scale=2):
366
  with gr.Row():
367
  with gr.Column():
368
- gr.Markdown("## πŸ“€ Medical Image Upload")
369
  image_input = gr.Image(
370
  label="Upload Medical Image",
371
  type="pil",
372
- height=350,
373
- sources=["upload", "clipboard"]
374
  )
375
 
376
  with gr.Column():
377
  gr.Markdown("## πŸ’¬ Clinical Information")
378
  clinical_question = gr.Textbox(
379
  label="Clinical Question *",
380
- placeholder="Examples:\nβ€’ Analyze this chest X-ray comprehensively\nβ€’ What pathological findings are visible in this image?\nβ€’ Provide differential diagnosis based on these imaging findings\nβ€’ Describe abnormalities and their clinical significance\nβ€’ Evaluate this medical image systematically",
381
- lines=5,
382
- max_lines=8
383
  )
384
 
385
  patient_history = gr.Textbox(
386
- label="Patient History & Clinical Context (Optional)",
387
- placeholder="e.g., 58-year-old female presenting with chest pain and shortness of breath. History of hypertension and smoking. Recent onset of symptoms.",
388
- lines=3,
389
- max_lines=5
390
  )
391
 
392
  with gr.Row():
393
- clear_btn = gr.Button("πŸ—‘οΈ Clear All", variant="secondary")
394
  analyze_btn = gr.Button("πŸ” Analyze with LLaVA", variant="primary", size="lg")
395
 
396
- gr.Markdown("## πŸ“‹ LLaVA Medical Analysis Results")
397
  output = gr.Textbox(
398
- label="Comprehensive Medical Analysis",
399
- lines=30,
400
- max_lines=50,
401
  show_copy_button=True,
402
- placeholder="Upload a medical image and provide a clinical question to receive comprehensive AI-powered medical analysis..." if llava_ready else "LLaVA model is loading. Please wait and refresh the page."
403
  )
404
 
405
- # Right column - System info and controls
406
  with gr.Column(scale=1):
407
  gr.Markdown("## ℹ️ System Status")
408
 
409
- model_status = "βœ… Ready" if llava_ready else "πŸ”„ Loading"
410
- device_info = "GPU" if torch.cuda.is_available() else "CPU"
411
 
412
  gr.Markdown(f"""
413
- **Model Status:** {model_status}
414
  **AI Model:** LLaVA-v1.6-Mistral-7B
415
- **Device:** {device_info}
416
- **Capabilities:** Medical image analysis + clinical reasoning
417
- **Context Length:** 32K tokens
418
- **Rate Limit:** 30 requests/hour
419
  """)
420
 
421
- gr.Markdown("## πŸ“Š Usage Analytics")
422
  stats_display = gr.Markdown("")
423
- refresh_stats_btn = gr.Button("πŸ”„ Refresh Statistics", size="sm")
424
 
425
  if llava_ready:
426
- gr.Markdown("## 🎯 Quick Clinical Examples")
427
-
428
- radiology_btn = gr.Button("🫁 Chest X-ray Analysis", size="sm")
429
- pathology_btn = gr.Button("πŸ”¬ Pathology Review", size="sm")
430
- dermatology_btn = gr.Button("🩺 Skin Lesion Analysis", size="sm")
431
- differential_btn = gr.Button("🧠 Differential Diagnosis", size="sm")
432
-
433
- gr.Markdown("## πŸ₯ Medical Specialties")
434
- gr.Markdown("""
435
- **LLaVA excels in:**
436
- - Radiology interpretation
437
- - Pathological analysis
438
- - Dermatological assessment
439
- - Ophthalmological evaluation
440
- - Clinical reasoning & education
441
- """)
442
-
443
- # Comprehensive example cases
444
  if llava_ready:
445
- with gr.Accordion("πŸ“š Sample Medical Cases & Examples", open=False):
446
  examples = gr.Examples(
447
  examples=[
448
  [
449
  "https://upload.wikimedia.org/wikipedia/commons/c/c8/Chest_Xray_PA_3-8-2010.png",
450
- "Please perform a comprehensive systematic analysis of this chest X-ray. Evaluate image quality, assess cardiac silhouette, examine lung fields bilaterally, review mediastinal structures, and identify any pathological findings. Provide differential diagnosis if abnormalities are present.",
451
- "Adult patient presenting with acute onset chest pain and shortness of breath. No significant past medical history."
452
- ],
453
- [
454
- None,
455
- "Analyze this medical image systematically. Describe normal anatomical structures, identify any abnormal findings, assess clinical significance, and provide appropriate differential diagnoses based on imaging characteristics.",
456
- "Patient with acute presentation requiring medical imaging evaluation"
457
- ],
458
- [
459
- None,
460
- "What pathological changes are visible in this medical image? Please provide detailed morphological analysis, clinical correlation, and discuss potential diagnoses with supporting evidence from the imaging findings.",
461
- ""
462
  ]
463
  ],
464
- inputs=[image_input, clinical_question, patient_history],
465
- label="Click any example to load it into the interface"
466
  )
467
 
468
  # Event handlers
@@ -473,11 +511,8 @@ def create_interface():
473
  show_progress=True
474
  )
475
 
476
- def clear_all_fields():
477
- return None, "", "", ""
478
-
479
  clear_btn.click(
480
- fn=clear_all_fields,
481
  outputs=[image_input, clinical_question, patient_history, output]
482
  )
483
 
@@ -486,76 +521,37 @@ def create_interface():
486
  outputs=stats_display
487
  )
488
 
489
- # Quick example button handlers
490
  if llava_ready:
491
- radiology_btn.click(
492
- fn=lambda: ("Perform systematic radiological analysis of this medical image. Assess technical quality, identify normal anatomical structures, detect any pathological findings, and provide clinical interpretation with differential diagnosis.", "Adult patient with respiratory symptoms"),
493
- outputs=[clinical_question, patient_history]
494
- )
495
-
496
- pathology_btn.click(
497
- fn=lambda: ("Analyze this pathological specimen or medical image. Describe morphological features, identify cellular patterns, assess for pathological changes, and provide histopathological interpretation with clinical significance.", "Tissue sample for pathological evaluation"),
498
  outputs=[clinical_question, patient_history]
499
  )
500
 
501
- dermatology_btn.click(
502
- fn=lambda: ("Examine this dermatological image systematically. Describe the lesion characteristics, assess morphological features, evaluate for concerning signs, and provide differential diagnosis with clinical recommendations.", "Patient presenting with skin lesion requiring evaluation"),
503
  outputs=[clinical_question, patient_history]
504
  )
505
 
506
- differential_btn.click(
507
- fn=lambda: ("Based on the imaging findings in this medical image, provide a comprehensive differential diagnosis. List possible conditions in order of likelihood, explain supporting evidence for each diagnosis, and recommend additional studies if needed.", "Patient requiring diagnostic workup based on imaging findings"),
508
  outputs=[clinical_question, patient_history]
509
  )
510
 
511
- # Comprehensive footer with detailed information
512
  gr.Markdown("""
513
  ---
514
- ## πŸ€– About LLaVA Medical AI
515
-
516
- **LLaVA (Large Language and Vision Assistant)** is a state-of-the-art multimodal AI model that combines advanced computer vision with natural language processing for comprehensive medical image analysis.
517
-
518
- ### πŸ”¬ Key Capabilities
519
-
520
- **Medical Image Analysis:**
521
- - **Radiology**: X-rays, CT scans, MRI, ultrasound interpretation
522
- - **Pathology**: Histological analysis, tissue examination, cellular morphology
523
- - **Dermatology**: Skin lesion analysis, dermatological condition assessment
524
- - **Ophthalmology**: Retinal imaging, ocular pathology evaluation
525
 
526
- **Clinical Reasoning:**
527
- - Systematic medical image interpretation
528
- - Differential diagnosis generation
529
- - Clinical correlation and significance assessment
530
- - Educational medical content and explanations
531
 
532
- ### πŸ₯ Medical Education Applications
 
 
 
 
533
 
534
- - **Medical Student Training**: Interactive case-based learning
535
- - **Resident Education**: Systematic approach to image interpretation
536
- - **Continuing Medical Education**: Advanced diagnostic reasoning
537
- - **Research Applications**: Medical imaging analysis and documentation
538
-
539
- ### πŸ”’ Privacy & Compliance
540
-
541
- - **No Data Storage**: All images processed in real-time, not stored
542
- - **Educational Purpose**: Designed specifically for medical education and training
543
- - **Privacy Protection**: No patient identifiable information should be uploaded
544
- - **Professional Standards**: Adheres to medical AI ethics and best practices
545
-
546
- ### ⚑ Technical Specifications
547
-
548
- - **Model**: LLaVA-v1.6-Mistral-7B (Latest version)
549
- - **Context Window**: 32,000 tokens for comprehensive analysis
550
- - **Processing**: Real-time inference with detailed medical reasoning
551
- - **Accuracy**: Research-grade performance on medical imaging tasks
552
-
553
- ### πŸ“ž Support & Resources
554
-
555
- For technical support, feature requests, or educational partnerships, please contact our support team.
556
-
557
- ---
558
- **Powered by**: LLaVA (Large Language and Vision Assistant) | **License**: Apache 2.0 | **Purpose**: Medical Education & Research
559
  """)
560
 
561
  return demo
 
1
+ # app.py - Fixed LLaVA Medical AI with NoneType Error Resolution
2
  import gradio as gr
3
  import torch
 
 
4
  import logging
5
  from collections import defaultdict, Counter
6
  import time
7
+ import traceback
8
 
9
  # Configure logging
10
  logging.basicConfig(level=logging.INFO)
11
  logger = logging.getLogger(__name__)
12
 
13
+ # Fix the NoneType compatibility issue
14
+ def fix_transformers_compatibility():
15
+ """Fix compatibility issues with transformers library"""
16
+ try:
17
+ # Import and fix the parallel styles issue
18
+ import transformers.modeling_utils as modeling_utils
19
+ if not hasattr(modeling_utils, 'ALL_PARALLEL_STYLES'):
20
+ modeling_utils.ALL_PARALLEL_STYLES = []
21
+ elif getattr(modeling_utils, 'ALL_PARALLEL_STYLES', None) is None:
22
+ modeling_utils.ALL_PARALLEL_STYLES = []
23
+
24
+ # Fix in specific model files
25
+ try:
26
+ import transformers.models.llava_next.modeling_llava_next as llava_next
27
+ if not hasattr(llava_next, 'ALL_PARALLEL_STYLES'):
28
+ llava_next.ALL_PARALLEL_STYLES = []
29
+ elif getattr(llava_next, 'ALL_PARALLEL_STYLES', None) is None:
30
+ llava_next.ALL_PARALLEL_STYLES = []
31
+ except ImportError:
32
+ pass
33
+
34
+ # Fix in mistral files if they exist
35
+ try:
36
+ import transformers.models.mistral.modeling_mistral as mistral
37
+ if not hasattr(mistral, 'ALL_PARALLEL_STYLES'):
38
+ mistral.ALL_PARALLEL_STYLES = []
39
+ elif getattr(mistral, 'ALL_PARALLEL_STYLES', None) is None:
40
+ mistral.ALL_PARALLEL_STYLES = []
41
+ except ImportError:
42
+ pass
43
+
44
+ logger.info("βœ… Applied compatibility fixes")
45
+ return True
46
+ except Exception as e:
47
+ logger.warning(f"⚠️ Could not apply compatibility fixes: {e}")
48
+ return False
49
+
50
+ # Apply compatibility fix before imports
51
+ fix_transformers_compatibility()
52
+
53
+ # Now import transformers
54
+ try:
55
+ from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration
56
+ from PIL import Image
57
+ logger.info("βœ… Transformers imported successfully")
58
+ except Exception as e:
59
+ logger.error(f"❌ Failed to import transformers: {e}")
60
+ # Fallback imports
61
+ try:
62
+ from transformers import LlavaProcessor, LlavaForConditionalGeneration as LlavaNextForConditionalGeneration
63
+ from transformers import AutoProcessor as LlavaNextProcessor
64
+ logger.info("βœ… Using fallback LLaVA imports")
65
+ except Exception as e2:
66
+ logger.error(f"❌ Fallback imports also failed: {e2}")
67
+
68
  # Usage tracking
69
  class UsageTracker:
70
  def __init__(self):
 
91
 
92
  # Rate limiting
93
  class RateLimiter:
94
+ def __init__(self, max_requests_per_hour=20):
95
  self.max_requests_per_hour = max_requests_per_hour
96
  self.requests = defaultdict(list)
97
 
 
108
  usage_tracker = UsageTracker()
109
  rate_limiter = RateLimiter()
110
 
111
+ # Model configuration
112
  MODEL_ID = "llava-hf/llava-v1.6-mistral-7b-hf"
113
 
114
  # Global variables
115
  model = None
116
  processor = None
117
 
118
+ def load_llava_safe():
119
+ """Load LLaVA model with comprehensive error handling"""
120
  global model, processor
121
 
122
  try:
123
  logger.info(f"Loading LLaVA model: {MODEL_ID}")
124
 
125
+ # Try different loading approaches
126
+ loading_methods = [
127
+ ("Standard LlavaNext", lambda: (
128
+ LlavaNextProcessor.from_pretrained(MODEL_ID),
129
+ LlavaNextForConditionalGeneration.from_pretrained(
130
+ MODEL_ID,
131
+ torch_dtype=torch.float32, # Use float32 for stability
132
+ device_map=None, # Let PyTorch handle device placement
133
+ low_cpu_mem_usage=True,
134
+ attn_implementation="eager" # Use eager attention to avoid issues
135
+ )
136
+ )),
137
+ ("Auto Processor Fallback", lambda: (
138
+ LlavaNextProcessor.from_pretrained(MODEL_ID),
139
+ LlavaNextForConditionalGeneration.from_pretrained(
140
+ MODEL_ID,
141
+ torch_dtype=torch.float32,
142
+ trust_remote_code=True,
143
+ use_safetensors=True
144
+ )
145
+ )),
146
+ ]
147
 
148
+ for method_name, method_func in loading_methods:
149
+ try:
150
+ logger.info(f"Trying {method_name}...")
151
+ processor, model = method_func()
152
+ logger.info(f"βœ… LLaVA loaded successfully using {method_name}!")
153
+ return True
154
+ except Exception as e:
155
+ logger.warning(f"❌ {method_name} failed: {str(e)}")
156
+ continue
157
+
158
+ logger.error("❌ All loading methods failed")
159
+ return False
160
 
161
  except Exception as e:
162
  logger.error(f"❌ Error loading LLaVA: {str(e)}")
163
+ logger.error(f"Full traceback: {traceback.format_exc()}")
164
  return False
165
 
166
  # Load model at startup
167
+ llava_ready = load_llava_safe()
168
 
169
  def analyze_medical_image_llava(image, clinical_question, patient_history=""):
170
+ """Analyze medical image using LLaVA with robust error handling"""
171
  start_time = time.time()
172
 
173
  # Rate limiting
 
177
 
178
  if not llava_ready or model is None:
179
  usage_tracker.log_analysis(False, time.time() - start_time)
180
+ return """❌ **LLaVA Model Loading Issue**
181
+
182
+ The LLaVA model failed to load due to compatibility issues. This is often caused by:
183
+
184
+ 1. **Library Version Conflicts**: Try refreshing the page - we've applied compatibility fixes
185
+ 2. **Memory Constraints**: The 7B model requires significant resources
186
+ 3. **Transformers Version**: Some versions have compatibility issues
187
+
188
+ **Suggested Solutions:**
189
+ - **Refresh the page** and wait 2-3 minutes for model loading
190
+ - **Upgrade to GPU hardware** for better performance and stability
191
+ - **Try a different image** if the issue persists
192
+
193
+ **Technical Info**: There may be version conflicts in the transformers library. The model files downloaded successfully but initialization failed."""
194
 
195
  if image is None:
196
  return "⚠️ Please upload a medical image first."
 
201
  try:
202
  logger.info("Starting LLaVA medical analysis...")
203
 
204
+ # Prepare medical prompt
205
+ medical_prompt = f"""You are an expert medical AI assistant analyzing medical images. Please provide a comprehensive medical analysis.
206
+
207
+ {f"Patient History: {patient_history}" if patient_history.strip() else ""}
208
+
209
+ Clinical Question: {clinical_question}
210
+
211
+ Please analyze this medical image systematically:
212
+
213
+ 1. **Image Quality**: Assess technical quality and diagnostic adequacy
214
+ 2. **Anatomical Structures**: Identify visible normal structures
215
+ 3. **Abnormal Findings**: Describe any pathological changes
216
+ 4. **Clinical Significance**: Explain the importance of findings
217
+ 5. **Assessment**: Provide clinical interpretation
218
+ 6. **Recommendations**: Suggest next steps if appropriate
219
+
220
+ Provide detailed, educational medical analysis suitable for learning purposes."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
221
 
222
+ # Different prompt formats to try
223
+ prompt_formats = [
224
+ # Format 1: Simple user message
225
+ lambda: f"USER: <image>\n{medical_prompt}\nASSISTANT:",
226
+
227
+ # Format 2: Chat format
228
+ lambda: processor.apply_chat_template([
229
+ {"role": "user", "content": [
230
+ {"type": "image", "image": image},
231
+ {"type": "text", "text": medical_prompt}
232
+ ]}
233
+ ], add_generation_prompt=True),
234
+
235
+ # Format 3: Direct format
236
+ lambda: medical_prompt
237
+ ]
 
 
 
 
 
 
238
 
239
+ # Try different prompt formats
240
+ for i, prompt_func in enumerate(prompt_formats):
241
+ try:
242
+ logger.info(f"Trying prompt format {i+1}...")
243
+
244
+ if i == 1: # Chat template format
245
+ try:
246
+ prompt = prompt_func()
247
+ except:
248
+ continue
249
+ else:
250
+ prompt = prompt_func()
251
+
252
+ # Process inputs
253
+ inputs = processor(prompt, image, return_tensors='pt')
254
+
255
+ # Generate response with conservative settings
256
+ logger.info("Generating medical analysis...")
257
+ with torch.inference_mode():
258
+ output = model.generate(
259
+ **inputs,
260
+ max_new_tokens=1000, # Conservative limit
261
+ do_sample=True,
262
+ temperature=0.3,
263
+ top_p=0.9,
264
+ repetition_penalty=1.1,
265
+ use_cache=False # Disable cache for stability
266
+ )
267
+
268
+ # Decode response
269
+ generated_text = processor.decode(output[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True)
270
+
271
+ if generated_text and generated_text.strip():
272
+ break
273
+
274
+ except Exception as e:
275
+ logger.warning(f"Prompt format {i+1} failed: {e}")
276
+ if i == len(prompt_formats) - 1: # Last attempt
277
+ raise e
278
+ continue
279
 
280
  # Clean up response
281
+ response = generated_text.strip() if generated_text else "Analysis completed."
282
 
283
+ # Format the response
284
+ formatted_response = f"""# πŸ₯ **LLaVA Medical Analysis**
285
 
286
  ## **Clinical Question:** {clinical_question}
287
  {f"## **Patient History:** {patient_history}" if patient_history.strip() else ""}
288
 
289
  ---
290
 
291
+ ## πŸ” **Medical Analysis Results**
292
 
293
  {response}
294
 
295
  ---
296
 
297
+ ## πŸ“‹ **Clinical Summary**
298
+
299
+ This analysis was generated using LLaVA (Large Language and Vision Assistant) for educational purposes. The findings should be interpreted by qualified medical professionals and correlated with clinical presentation.
300
 
301
  **Key Points:**
302
+ - Analysis based on visual medical image interpretation
303
+ - Systematic approach to medical imaging assessment
304
+ - Educational tool for medical learning and training
305
+ - Requires professional medical validation
 
 
 
 
 
 
 
 
 
306
  """
307
 
308
+ # Add medical disclaimer
309
  disclaimer = """
310
  ---
311
+ ## ⚠️ **MEDICAL DISCLAIMER**
312
 
313
+ **FOR EDUCATIONAL PURPOSES ONLY**
314
 
315
+ - **Not Diagnostic**: This AI analysis is not a medical diagnosis
316
+ - **Professional Review**: All findings require validation by healthcare professionals
317
+ - **Emergency Care**: Contact emergency services for urgent medical concerns
318
+ - **Educational Tool**: Designed for medical education and training
319
+ - **No PHI**: Do not upload patient identifiable information
 
 
 
 
320
 
321
  ---
322
+ **Powered by**: LLaVA (Large Language and Vision Assistant)
323
+ """
324
 
325
  # Log successful analysis
326
  duration = time.time() - start_time
 
335
  usage_tracker.log_analysis(False, duration)
336
  logger.error(f"❌ LLaVA analysis error: {str(e)}")
337
 
338
+ return f"""❌ **Analysis Error**
339
+
340
+ The analysis failed with error: {str(e)}
341
+
342
+ **Common Solutions:**
343
+ - **Try again**: Sometimes temporary processing issues occur
344
+ - **Smaller image**: Try with a smaller or different format image
345
+ - **Simpler question**: Use a more straightforward clinical question
346
+ - **Refresh page**: Reload the page if model seems unstable
347
+
348
+ **Technical Details:** {str(e)[:200]}"""
349
 
350
  def classify_question(question):
351
  """Classify clinical question type"""
352
  question_lower = question.lower()
353
+ if any(word in question_lower for word in ['describe', 'findings', 'observe']):
354
  return 'descriptive'
355
+ elif any(word in question_lower for word in ['diagnosis', 'differential', 'condition']):
356
  return 'diagnostic'
357
+ elif any(word in question_lower for word in ['abnormal', 'pathology', 'disease']):
358
  return 'pathological'
 
 
359
  else:
360
  return 'general'
361
 
362
  def get_usage_stats():
363
+ """Get usage statistics"""
364
  stats = usage_tracker.stats
365
  if stats['total_analyses'] == 0:
366
  return "πŸ“Š **Usage Statistics**\n\nNo analyses performed yet."
367
 
368
  success_rate = (stats['successful_analyses'] / stats['total_analyses']) * 100
369
 
370
+ return f"""πŸ“Š **LLaVA Usage Statistics**
371
+
372
+ **Performance:**
373
+ - Total Analyses: {stats['total_analyses']}
374
+ - Success Rate: {success_rate:.1f}%
375
+ - Avg Processing Time: {stats['average_processing_time']:.2f}s
376
+
377
+ **Popular Question Types:**
378
+ {chr(10).join([f"- {qtype}: {count}" for qtype, count in stats['question_types'].most_common(3)])}
379
+
380
+ **Model Status**: {'🟒 Ready' if llava_ready else 'πŸ”΄ Loading Issues'}
 
 
 
 
 
381
  """
382
 
383
+ # Create Gradio interface
384
  def create_interface():
385
  with gr.Blocks(
386
+ title="LLaVA Medical Analysis",
387
  theme=gr.themes.Soft(),
388
  css="""
389
+ .gradio-container { max-width: 1200px !important; }
390
  .disclaimer { background-color: #fef2f2; border: 1px solid #fecaca; border-radius: 8px; padding: 16px; margin: 16px 0; }
391
  .success { background-color: #f0f9ff; border: 1px solid #bae6fd; border-radius: 8px; padding: 16px; margin: 16px 0; }
392
  .warning { background-color: #fffbeb; border: 1px solid #fed7aa; border-radius: 8px; padding: 16px; margin: 16px 0; }
 
399
 
400
  **Advanced Medical AI powered by LLaVA (Large Language and Vision Assistant)**
401
 
402
+ **Medical Capabilities:** 🫁 Radiology β€’ πŸ”¬ Pathology β€’ 🩺 Dermatology β€’ πŸ‘οΈ Ophthalmology
 
403
  """)
404
 
405
  # Status display
 
407
  gr.Markdown("""
408
  <div class="success">
409
  βœ… <strong>LLAVA MEDICAL AI READY</strong><br>
410
+ LLaVA model loaded successfully with compatibility fixes. Ready for medical image analysis.
411
  </div>
412
  """)
413
  else:
414
  gr.Markdown("""
415
  <div class="warning">
416
+ ⚠️ <strong>MODEL LOADING ISSUE</strong><br>
417
+ LLaVA model had loading problems. Try refreshing the page or contact support for assistance.
418
  </div>
419
  """)
420
 
421
  # Medical disclaimer
422
  gr.Markdown("""
423
  <div class="disclaimer">
424
+ ⚠️ <strong>MEDICAL DISCLAIMER</strong><br>
425
+ This AI provides medical analysis for <strong>educational purposes only</strong>.
426
+ Do not upload real patient data. Always consult healthcare professionals for medical decisions.
 
427
  </div>
428
  """)
429
 
430
  with gr.Row():
431
+ # Left column
432
  with gr.Column(scale=2):
433
  with gr.Row():
434
  with gr.Column():
435
+ gr.Markdown("## πŸ“€ Medical Image")
436
  image_input = gr.Image(
437
  label="Upload Medical Image",
438
  type="pil",
439
+ height=300
 
440
  )
441
 
442
  with gr.Column():
443
  gr.Markdown("## πŸ’¬ Clinical Information")
444
  clinical_question = gr.Textbox(
445
  label="Clinical Question *",
446
+ placeholder="Examples:\nοΏ½οΏ½ Analyze this medical image\nβ€’ What abnormalities are visible?\nβ€’ Describe the findings\nβ€’ Provide medical interpretation",
447
+ lines=4
 
448
  )
449
 
450
  patient_history = gr.Textbox(
451
+ label="Patient History (Optional)",
452
+ placeholder="e.g., 45-year-old with chest pain",
453
+ lines=2
 
454
  )
455
 
456
  with gr.Row():
457
+ clear_btn = gr.Button("πŸ—‘οΈ Clear", variant="secondary")
458
  analyze_btn = gr.Button("πŸ” Analyze with LLaVA", variant="primary", size="lg")
459
 
460
+ gr.Markdown("## πŸ“‹ Medical Analysis Results")
461
  output = gr.Textbox(
462
+ label="LLaVA Medical Analysis",
463
+ lines=20,
 
464
  show_copy_button=True,
465
+ placeholder="Upload a medical image and clinical question..." if llava_ready else "Model loading issues - please refresh the page"
466
  )
467
 
468
+ # Right column
469
  with gr.Column(scale=1):
470
  gr.Markdown("## ℹ️ System Status")
471
 
472
+ status = "βœ… Ready" if llava_ready else "⚠️ Loading Issues"
 
473
 
474
  gr.Markdown(f"""
475
+ **Model Status:** {status}
476
  **AI Model:** LLaVA-v1.6-Mistral-7B
477
+ **Device:** {'GPU' if torch.cuda.is_available() else 'CPU'}
478
+ **Compatibility:** Fixed for stability
479
+ **Rate Limit:** 20 requests/hour
 
480
  """)
481
 
482
+ gr.Markdown("## πŸ“Š Usage Statistics")
483
  stats_display = gr.Markdown("")
484
+ refresh_stats_btn = gr.Button("πŸ”„ Refresh Stats", size="sm")
485
 
486
  if llava_ready:
487
+ gr.Markdown("## 🎯 Quick Examples")
488
+ general_btn = gr.Button("General Analysis", size="sm")
489
+ findings_btn = gr.Button("Find Abnormalities", size="sm")
490
+ interpret_btn = gr.Button("Medical Interpretation", size="sm")
491
+
492
+ # Example cases
 
 
 
 
 
 
 
 
 
 
 
 
493
  if llava_ready:
494
+ with gr.Accordion("πŸ“š Example Cases", open=False):
495
  examples = gr.Examples(
496
  examples=[
497
  [
498
  "https://upload.wikimedia.org/wikipedia/commons/c/c8/Chest_Xray_PA_3-8-2010.png",
499
+ "Please analyze this chest X-ray and describe any findings. Assess the image quality, identify normal structures, and note any abnormalities.",
500
+ "Adult patient with respiratory symptoms"
 
 
 
 
 
 
 
 
 
 
501
  ]
502
  ],
503
+ inputs=[image_input, clinical_question, patient_history]
 
504
  )
505
 
506
  # Event handlers
 
511
  show_progress=True
512
  )
513
 
 
 
 
514
  clear_btn.click(
515
+ fn=lambda: (None, "", "", ""),
516
  outputs=[image_input, clinical_question, patient_history, output]
517
  )
518
 
 
521
  outputs=stats_display
522
  )
523
 
524
+ # Quick example handlers
525
  if llava_ready:
526
+ general_btn.click(
527
+ fn=lambda: ("Analyze this medical image comprehensively. Describe what you observe and provide medical interpretation.", ""),
 
 
 
 
 
528
  outputs=[clinical_question, patient_history]
529
  )
530
 
531
+ findings_btn.click(
532
+ fn=lambda: ("What abnormalities or pathological findings are visible in this medical image?", ""),
533
  outputs=[clinical_question, patient_history]
534
  )
535
 
536
+ interpret_btn.click(
537
+ fn=lambda: ("Provide medical interpretation of this image including clinical significance of any findings.", ""),
538
  outputs=[clinical_question, patient_history]
539
  )
540
 
541
+ # Footer
542
  gr.Markdown("""
543
  ---
544
+ ### πŸ€– LLaVA Medical AI
 
 
 
 
 
 
 
 
 
 
545
 
546
+ **Large Language and Vision Assistant** optimized for medical image analysis with compatibility fixes for stable operation.
 
 
 
 
547
 
548
+ **Features:**
549
+ - Advanced medical image interpretation
550
+ - Systematic clinical analysis approach
551
+ - Educational medical explanations
552
+ - Comprehensive error handling
553
 
554
+ **Model:** LLaVA-v1.6-Mistral-7B | **Purpose:** Medical Education & Research
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
555
  """)
556
 
557
  return demo