initialized the first deployment

Dockerfile

@@ -0,0 +1,20 @@
+# Use the official Python 3.9 image
+FROM python:3.9
+
+# Create a non-root user and switch to it
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+# Set working directory
+WORKDIR /app
+
+# Copy requirements and install
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# Copy the rest of the app
+COPY --chown=user . /app
+
+# Expose port 7860 and run the app with uvicorn
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -0,0 +1,392 @@
+# app.py
+import os, io, base64, cv2, torch, numpy as np
+from PIL import Image
+from flask import Flask, request, render_template, jsonify
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision.models as models
+import torchvision.transforms as transforms
+from monai.transforms import EnsureChannelFirst, ScaleIntensity, Resize, ToTensor
+
+# Enable debug logging
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+# -------------------------------
+# Global Setup
+# -------------------------------
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+def pil_to_base64(pil_img):
+    buff = io.BytesIO()
+    pil_img.save(buff, format="JPEG")
+    return base64.b64encode(buff.getvalue()).decode("utf-8")
+
+# -------------------------------
+# 1. CLASSIFIER MODULE (DenseNet121 via MONAI)
+# -------------------------------
+CLASS_NAMES = ['AbdomenCT', 'BreastMRI', 'Chest Xray', 'ChestCT',
+               'Endoscopy', 'Hand Xray', 'HeadCT', 'HeadMRI']
+from monai.networks.nets import DenseNet121
+def load_classifier_model(model_path):
+    model = DenseNet121(
+        spatial_dims=2,
+        in_channels=3,
+        out_channels=len(CLASS_NAMES)
+    ).to(device)
+    state_dict = torch.load(model_path, map_location=device)
+    if isinstance(state_dict, dict) and "state_dict" in state_dict:
+        state_dict = state_dict["state_dict"]
+    model.load_state_dict(state_dict, strict=False)
+    model.eval()
+    return model
+
+def load_and_preprocess_image_classifier(image_path):
+    image_path = image_path.strip()
+    if image_path.lower().endswith((".jpg", ".jpeg", ".png")):
+        image = Image.open(image_path).convert("RGB")
+        image = np.array(image)
+    elif image_path.lower().endswith((".nii", ".nii.gz")):
+        import nibabel as nib
+        image = nib.load(image_path).get_fdata()
+        image = np.squeeze(image)
+        if len(image.shape) == 4:
+            image = image[..., 0]
+        if len(image.shape) == 3:
+            image = image[:, :, image.shape[2] // 2]
+        if len(image.shape) == 2:
+            image = np.stack([image]*3, axis=-1)
+    elif image_path.lower().endswith(".dcm"):
+        import pydicom
+        dicom_data = pydicom.dcmread(image_path)
+        image = dicom_data.pixel_array
+        if len(image.shape) == 2:
+            image = np.stack([image]*3, axis=-1)
+    else:
+        raise ValueError("Unsupported file format!")
+    if len(image.shape) == 3 and image.shape[-1] == 3:
+        image = np.transpose(image, (2, 0, 1))
+    else:
+        raise ValueError(f"Unexpected image shape: {image.shape}")
+    image = torch.tensor(image, dtype=torch.float32)
+    image = ScaleIntensity()(image)
+    image = Resize((224,224))(image)
+    image = image.unsqueeze(0)
+    return image.to(device)
+
+def classify_medical_image(image_path, classifier_model):
+    image_tensor = load_and_preprocess_image_classifier(image_path)
+    with torch.no_grad():
+        output = classifier_model(image_tensor)
+        pred_class = torch.argmax(output, dim=1).item()
+    return CLASS_NAMES[pred_class]
+
+# -------------------------------
+# 2. BRAIN TUMOR SEGMENTATION MODULE (UNetMulti)
+# -------------------------------
+class DoubleConvUNet(nn.Module):
+    def __init__(self, in_channels, out_channels):
+        super(DoubleConvUNet, self).__init__()
+        self.conv = nn.Sequential(
+            nn.Conv2d(in_channels, out_channels, 3, padding=1),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(out_channels, out_channels, 3, padding=1),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU(inplace=True)
+        )
+    def forward(self, x):
+        return self.conv(x)
+
+class UNetMulti(nn.Module):
+    def __init__(self, in_channels=3, out_channels=4):
+        super(UNetMulti, self).__init__()
+        self.down1 = DoubleConvUNet(in_channels, 64)
+        self.pool1 = nn.MaxPool2d(2)
+        self.down2 = DoubleConvUNet(64, 128)
+        self.pool2 = nn.MaxPool2d(2)
+        self.down3 = DoubleConvUNet(128, 256)
+        self.pool3 = nn.MaxPool2d(2)
+        self.down4 = DoubleConvUNet(256, 512)
+        self.pool4 = nn.MaxPool2d(2)
+        self.bottleneck = DoubleConvUNet(512, 1024)
+        self.up4 = nn.ConvTranspose2d(1024, 512, 2, stride=2)
+        self.conv4 = DoubleConvUNet(1024, 512)
+        self.up3 = nn.ConvTranspose2d(512, 256, 2, stride=2)
+        self.conv3 = DoubleConvUNet(512, 256)
+        self.up2 = nn.ConvTranspose2d(256, 128, 2, stride=2)
+        self.conv2 = DoubleConvUNet(256, 128)
+        self.up1 = nn.ConvTranspose2d(128, 64, 2, stride=2)
+        self.conv1 = DoubleConvUNet(128, 64)
+        self.final_conv = nn.Conv2d(64, out_channels, 1)
+    def forward(self, x):
+        c1 = self.down1(x)
+        p1 = self.pool1(c1)
+        c2 = self.down2(p1)
+        p2 = self.pool2(c2)
+        c3 = self.down3(p2)
+        p3 = self.pool3(c3)
+        c4 = self.down4(p3)
+        p4 = self.pool4(c4)
+        bn = self.bottleneck(p4)
+        u4 = self.up4(bn)
+        merge4 = torch.cat([u4, c4], dim=1)
+        c5 = self.conv4(merge4)
+        u3 = self.up3(c5)
+        merge3 = torch.cat([u3, c3], dim=1)
+        c6 = self.conv3(merge3)
+        u2 = self.up2(c6)
+        merge2 = torch.cat([u2, c2], dim=1)
+        c7 = self.conv2(merge2)
+        u1 = self.up1(c7)
+        merge1 = torch.cat([u1, c1], dim=1)
+        c8 = self.conv1(merge1)
+        return self.final_conv(c8)
+
+def process_brain_tumor_return(image, model_path="models/brain_tumor_unet_multiclass.pth"):
+    logging.debug("Processing brain tumor segmentation")
+    model = UNetMulti(in_channels=3, out_channels=4).to(device)
+    model.load_state_dict(torch.load(model_path, map_location=device))
+    model.eval()
+    transform_img = transforms.Compose([
+        transforms.Resize((256,256)),
+        transforms.ToTensor()
+    ])
+    input_tensor = transform_img(image).unsqueeze(0).to(device)
+    with torch.no_grad():
+        output = model(input_tensor)
+        preds = torch.argmax(output, dim=1).squeeze().cpu().numpy()
+    image_np = transform_img(image).permute(1,2,0).cpu().numpy()
+    overlay = cv2.applyColorMap(np.uint8(255 * preds/np.max(preds+1e-8)), cv2.COLORMAP_JET)
+    overlay = cv2.cvtColor(overlay, cv2.COLOR_BGR2RGB)
+    blended = cv2.addWeighted(np.uint8(image_np*255), 0.6, overlay, 0.4, 0)
+    orig_pil = Image.fromarray((image_np*255).astype(np.uint8))
+    mask_pil = Image.fromarray(overlay)
+    overlay_pil = Image.fromarray(blended)
+    return {
+        "original": pil_to_base64(orig_pil),
+        "mask": pil_to_base64(mask_pil),
+        "overlay": pil_to_base64(overlay_pil)
+    }
+
+# -------------------------------
+# 3. ENDOSCOPY POLYP DETECTION MODULE (Binary UNet)
+# -------------------------------
+class UNetBinary(nn.Module):
+    def __init__(self, in_channels=3, out_channels=1):
+        super(UNetBinary, self).__init__()
+        self.down1 = DoubleConvUNet(in_channels, 64)
+        self.pool1 = nn.MaxPool2d(2)
+        self.down2 = DoubleConvUNet(64, 128)
+        self.pool2 = nn.MaxPool2d(2)
+        self.down3 = DoubleConvUNet(128, 256)
+        self.pool3 = nn.MaxPool2d(2)
+        self.down4 = DoubleConvUNet(256, 512)
+        self.pool4 = nn.MaxPool2d(2)
+        self.bottleneck = DoubleConvUNet(512, 1024)
+        self.up4 = nn.ConvTranspose2d(1024, 512, 2, stride=2)
+        self.conv4 = DoubleConvUNet(1024, 512)
+        self.up3 = nn.ConvTranspose2d(512, 256, 2, stride=2)
+        self.conv3 = DoubleConvUNet(512, 256)
+        self.up2 = nn.ConvTranspose2d(256, 128, 2, stride=2)
+        self.conv2 = DoubleConvUNet(256, 128)
+        self.up1 = nn.ConvTranspose2d(128, 64, 2, stride=2)
+        self.conv1 = DoubleConvUNet(128, 64)
+        self.final_conv = nn.Conv2d(64, out_channels, 1)
+    def forward(self, x):
+        c1 = self.down1(x)
+        p1 = self.pool1(c1)
+        c2 = self.down2(p1)
+        p2 = self.pool2(c2)
+        c3 = self.down3(p2)
+        p3 = self.pool3(c3)
+        c4 = self.down4(p3)
+        p4 = self.pool4(c4)
+        bn = self.bottleneck(p4)
+        u4 = self.up4(bn)
+        merge4 = torch.cat([u4, c4], dim=1)
+        c5 = self.conv4(merge4)
+        u3 = self.up3(c5)
+        merge3 = torch.cat([u3, c3], dim=1)
+        c6 = self.conv3(merge3)
+        u2 = self.up2(c6)
+        merge2 = torch.cat([u2, c2], dim=1)
+        c7 = self.conv2(merge2)
+        u1 = self.up1(c7)
+        merge1 = torch.cat([u1, c1], dim=1)
+        c8 = self.conv1(merge1)
+        return self.final_conv(c8)
+
+def process_endoscopy_return(image, model_path="models/endoscopy_unet.pth"):
+    model = UNetBinary(in_channels=3, out_channels=1).to(device)
+    model.load_state_dict(torch.load(model_path, map_location=device))
+    model.eval()
+    transform_img = transforms.Compose([
+        transforms.Resize((256,256)),
+        transforms.ToTensor()
+    ])
+    input_tensor = transform_img(image).unsqueeze(0).to(device)
+    with torch.no_grad():
+        output = model(input_tensor)
+        prob = torch.sigmoid(output)
+        mask = (prob > 0.5).float().squeeze().cpu().numpy()
+    image_np = transform_img(image).permute(1,2,0).cpu().numpy()
+    overlay = cv2.applyColorMap(np.uint8(255*mask), cv2.COLORMAP_JET)
+    overlay = cv2.cvtColor(overlay, cv2.COLOR_BGR2RGB)
+    blended = cv2.addWeighted(np.uint8(image_np*255), 0.6, overlay, 0.4, 0)
+    orig_pil = Image.fromarray((image_np*255).astype(np.uint8))
+    mask_pil = Image.fromarray(overlay)
+    overlay_pil = Image.fromarray(blended)
+    return {
+        "original": pil_to_base64(orig_pil),
+        "mask": pil_to_base64(mask_pil),
+        "overlay": pil_to_base64(overlay_pil)
+    }
+
+# -------------------------------
+# 4. PNEUMONIA DETECTION MODULE (Grad-CAM on ResNet18)
+# -------------------------------
+class GradCAM_Pneumonia:
+    def __init__(self, model, target_layer):
+        self.model = model
+        self.target_layer = target_layer
+        self.gradients = None
+        self.activations = None
+        self.hook_handles = []
+        self._register_hooks()
+    def _register_hooks(self):
+        def forward_hook(module, input, output):
+            self.activations = output.detach()
+        def backward_hook(module, grad_in, grad_out):
+            self.gradients = grad_out[0].detach()
+        handle1 = self.target_layer.register_forward_hook(forward_hook)
+        handle2 = self.target_layer.register_backward_hook(backward_hook)
+        self.hook_handles.extend([handle1, handle2])
+    def remove_hooks(self):
+        for handle in self.hook_handles:
+            handle.remove()
+    def generate(self, input_image, target_class=None):
+        output = self.model(input_image)
+        if target_class is None:
+            target_class = output.argmax(dim=1).item()
+        self.model.zero_grad()
+        one_hot = torch.zeros_like(output)
+        one_hot[0, target_class] = 1
+        with torch.enable_grad():
+            output.backward(gradient=one_hot, retain_graph=True)
+        weights = self.gradients.mean(dim=(2,3), keepdim=True)
+        cam = (weights * self.activations).sum(dim=1, keepdim=True)
+        cam = F.relu(cam)
+        cam = cam.squeeze().cpu().numpy()
+        _, _, H, W = input_image.shape
+        cam = cv2.resize(cam, (W, H))
+        cam = (cam - np.min(cam)) / (np.max(cam) - np.min(cam) + 1e-8)
+        return cam, output
+
+def process_pneumonia_return(image, model_path="models/pneumonia_resnet18.pth"):
+    model = models.resnet18(pretrained=False)
+    num_ftrs = model.fc.in_features
+    model.fc = nn.Linear(num_ftrs, 2)  # 2 classes: normal and pneumonia
+    model.load_state_dict(torch.load(model_path, map_location=device))
+    model.to(device)
+    model.eval()
+    grad_cam = GradCAM_Pneumonia(model, model.layer4)
+    
+    transform_img = transforms.Compose([
+        transforms.Resize((224,224)),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.485,0.456,0.406],
+                             std=[0.229,0.224,0.225])
+    ])
+    input_tensor = transform_img(image).unsqueeze(0).to(device)
+    # Enable gradient tracking for the input tensor
+    input_tensor.requires_grad_()  
+    # Do NOT wrap the following call with torch.no_grad()
+    cam, output = grad_cam.generate(input_tensor)
+    predicted_class = output.argmax(dim=1).item()
+    
+    label_text = "Pneumonia" if predicted_class == 1 else "Normal"
+    
+    def get_bounding_box(heatmap, thresh=0.5, min_area=100):
+        heat_uint8 = np.uint8(255 * heatmap)
+        ret, binary = cv2.threshold(heat_uint8, int(thresh*255), 255, cv2.THRESH_BINARY)
+        contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+        if len(contours)==0:
+            return None
+        largest = max(contours, key=cv2.contourArea)
+        if cv2.contourArea(largest) < min_area:
+            return None
+        x, y, w, h = cv2.boundingRect(largest)
+        return (x, y, w, h)
+    
+    bbox = None
+    if predicted_class == 1:
+        bbox = get_bounding_box(cam, thresh=0.5, min_area=100)
+    
+    resized_image = image.resize((224,224))
+    image_np = np.array(resized_image)
+    overlay = image_np.copy()
+    if bbox is not None:
+        x, y, w, h = bbox
+        cv2.rectangle(overlay, (x, y), (x+w, y+h), (255,0,0), 2)
+    cv2.putText(overlay, label_text, (10,25), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255,255,0),2)
+    
+    heatmap_color = cv2.applyColorMap(np.uint8(255*cam), cv2.COLORMAP_JET)
+    heatmap_color = cv2.cvtColor(heatmap_color, cv2.COLOR_BGR2RGB)
+    
+    orig_pil = Image.fromarray(image_np)
+    heatmap_pil = Image.fromarray(heatmap_color)
+    overlay_pil = Image.fromarray(overlay)
+    grad_cam.remove_hooks()
+    return {
+        "original": pil_to_base64(orig_pil),
+        "mask": pil_to_base64(heatmap_pil),
+        "overlay": pil_to_base64(overlay_pil)
+    }
+
+# -------------------------------
+# 5. COMPLETE PIPELINE FUNCTION
+# -------------------------------
+def complete_pipeline(image_path):
+    classifier_model = load_classifier_model("models/best_metric_model (4).pth")
+    predicted_modality = classify_medical_image(image_path, classifier_model)
+    print(f"Detected modality: {predicted_modality}")
+    original_image = Image.open(image_path).convert("RGB")
+    results = {"predicted_modality": predicted_modality}
+    if predicted_modality in ["HeadCT", "HeadMRI"]:
+        results["specialized"] = process_brain_tumor_return(original_image, "models/brain_tumor_unet_multiclass.pth")
+    elif predicted_modality == "Endoscopy":
+        results["specialized"] = process_endoscopy_return(original_image, "models/endoscopy_unet.pth")
+    elif predicted_modality == "Chest Xray":
+        results["specialized"] = process_pneumonia_return(original_image, "models/pneumonia_resnet18.pth")
+    else:
+        results["message"] = f"No specialized processing for modality: {predicted_modality}"
+    return results
+
+# -------------------------------
+# 6. FLASK API SETUP
+# -------------------------------
+from flask import Flask, request, render_template, jsonify
+app = Flask(__name__)
+
+@app.route('/', methods=['GET'])
+def index():
+    return render_template("index.html", result=None)
+
+@app.route('/predict', methods=['POST'])
+def predict():
+    if 'file' not in request.files:
+        return render_template("index.html", result={"error": "No file part in the request."})
+    file = request.files['file']
+    if file.filename == '':
+        return render_template("index.html", result={"error": "No file selected."})
+    temp_path = "temp_input.jpg"
+    file.save(temp_path)
+    try:
+        result = complete_pipeline(temp_path)
+    except Exception as e:
+        result = {"error": str(e)}
+    os.remove(temp_path)
+    return render_template("index.html", result=result)
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=5000, debug=True)

app/chatbot.py

@@ -0,0 +1,149 @@
+import pandas as pd
+import json
+import numpy as np
+import faiss
+from sklearn.feature_extraction.text import TfidfVectorizer
+from transformers import pipeline
+
+# -------------------------------
+# Load disease data and preprocess
+# -------------------------------
+def load_disease_data(csv_path):
+    df = pd.read_csv(csv_path)
+    df.columns = df.columns.str.strip().str.lower()
+    df = df.fillna("")
+    disease_symptoms = {}
+    disease_precautions = {}
+    for _, row in df.iterrows():
+        disease = row["disease"].strip()
+        symptoms = [s.strip().lower() for s in row["symptoms"].split(",") if s.strip()]
+        precautions = [p.strip() for p in row["precautions"].split(",") if p.strip()]
+        disease_symptoms[disease] = symptoms
+        disease_precautions[disease] = precautions
+    return disease_symptoms, disease_precautions
+
+# Load CSV data (ensure this CSV file is in the repository root)
+disease_symptoms, disease_precautions = load_disease_data("disease_sympts_prec_full.csv")
+known_symptoms = set()
+for syms in disease_symptoms.values():
+    known_symptoms.update(syms)
+
+# -------------------------------
+# Build symptom vectorizer and FAISS index
+# -------------------------------
+vectorizer = TfidfVectorizer()
+symptom_texts = [" ".join(symptoms) for symptoms in disease_symptoms.values()]
+tfidf_matrix = vectorizer.fit_transform(symptom_texts).toarray()
+index = faiss.IndexFlatL2(tfidf_matrix.shape[1])
+index.add(np.array(tfidf_matrix, dtype=np.float32))
+disease_list = list(disease_symptoms.keys())
+
+def find_closest_disease(user_symptoms):
+    if not user_symptoms:
+        return None
+    user_vector = vectorizer.transform([" ".join(user_symptoms)]).toarray().astype("float32")
+    distances, indices = index.search(user_vector, k=1)
+    return disease_list[indices[0][0]]
+
+# -------------------------------
+# Load Medical NER model for symptom extraction
+# -------------------------------
+medical_ner = pipeline(
+    "ner",
+    model="blaze999/Medical-NER",
+    tokenizer="blaze999/Medical-NER",
+    aggregation_strategy="simple"
+)
+
+def extract_symptoms_ner(text):
+    results = medical_ner(text)
+    extracted = []
+    for r in results:
+        if "SIGN_SYMPTOM" in r["entity_group"]:
+            extracted.append(r["word"].lower())
+    return list(set(extracted))
+
+def is_affirmative(answer):
+    answer_lower = answer.lower()
+    return any(word in answer_lower for word in ["yes", "yeah", "yep", "certainly", "sometimes", "a little"])
+
+# -------------------------------
+# Chatbot session class
+# -------------------------------
+class ChatbotSession:
+    def __init__(self):
+        self.conversation_history = []
+        self.reported_symptoms = set()
+        self.asked_missing = set()
+        self.awaiting_followup = None
+        self.state = "symptom_collection"  # states: symptom_collection, pain, medications
+        # Initial greeting
+        greeting = "Doctor: Hello, I am your virtual doctor. What brought you in today?"
+        self.conversation_history.append(greeting)
+        self.finished = False
+
+    def process_message(self, message: str) -> str:
+        # State: collecting symptoms
+        if self.state == "symptom_collection":
+            if message.lower() in ["exit", "quit", "no"]:
+                self.state = "pain"
+                prompt = "Doctor: Do you experience any pain or aches? Please rate the pain on a scale of 1 to 10 (or type 'no' if none):"
+                self.conversation_history.append(prompt)
+                return prompt
+            # If we are waiting on a follow-up about a specific symptom
+            if self.awaiting_followup:
+                if is_affirmative(message):
+                    self.reported_symptoms.add(self.awaiting_followup)
+                self.asked_missing.add(self.awaiting_followup)
+                self.awaiting_followup = None
+            else:
+                # Extract symptoms from message text
+                ner_results = extract_symptoms_ner(message)
+                for sym in ner_results:
+                    if sym not in self.reported_symptoms:
+                        self.reported_symptoms.add(sym)
+            # Update predicted disease
+            predicted_disease = find_closest_disease(list(self.reported_symptoms)) if self.reported_symptoms else None
+            # Check for missing symptoms if a disease is predicted
+            if predicted_disease:
+                expected = set(disease_symptoms.get(predicted_disease, []))
+                missing = expected - self.reported_symptoms
+                not_asked = missing - self.asked_missing
+                if not_asked:
+                    symptom_to_ask = list(not_asked)[0]
+                    followup = f"Are you also experiencing {symptom_to_ask}?"
+                    self.conversation_history.append("Doctor: " + followup)
+                    self.awaiting_followup = symptom_to_ask
+                    return followup
+            prompt = "Doctor: Do you have any other symptoms you'd like to mention?"
+            self.conversation_history.append(prompt)
+            return prompt
+
+        # State: asking about pain
+        elif self.state == "pain":
+            try:
+                self.pain_level = int(message)
+            except ValueError:
+                self.pain_level = message
+            self.state = "medications"
+            prompt = "Doctor: Have you taken any medications recently? If yes, please specify (or type 'no' if none):"
+            self.conversation_history.append(prompt)
+            return prompt
+
+        # State: asking about medications
+        elif self.state == "medications":
+            self.medications = message if message.lower() not in ["no", "none"] else "None"
+            closing = "Doctor: Thank you for providing all the information."
+            self.conversation_history.append(closing)
+            self.finished = True
+            return closing
+
+        return "Doctor: I'm sorry, I didn't understand that."
+
+    def get_data(self):
+        return {
+            "conversation": self.conversation_history,
+            "symptoms": list(self.reported_symptoms),
+            "pain_level": getattr(self, "pain_level", None),
+            "medications": getattr(self, "medications", None)
+        }

app/database.py

@@ -0,0 +1,14 @@
+import os
+from pymongo import MongoClient
+
+# Replace with your actual username and password
+MONGO_URI = "mongodb+srv://root:[email protected]/uspark_db?retryWrites=true&w=majority"
+
+client = MongoClient(MONGO_URI)
+db = client["uspark_db"]
+
+def save_chat_session(session_id: str, conversation_data: dict):
+    db.chatbot.insert_one({"session_id": session_id, **conversation_data})
+
+def save_medseg_result(result_data: dict):
+    db.medseg.insert_one(result_data)

app/main.py

@@ -0,0 +1,66 @@
+from fastapi import FastAPI, UploadFile, File, HTTPException, Form, Depends
+from uuid import uuid4
+import io
+from PIL import Image
+from pydantic import BaseModel
+
+# Import modules from the Uspark package
+from app.chatbot import ChatbotSession
+from app.mediseg import complete_pipeline_image
+from app.database import save_chat_session, save_medseg_result
+
+app = FastAPI(title="Uspark API")
+
+# Ensure models are loaded from the 'models' directory within 'Uspark'
+import sys
+import os
+sys.path.append(os.path.join(os.path.dirname(__file__), "../models"))
+
+class ChatMessage(BaseModel):
+    session_id: str
+    message: str
+
+# In-memory session store (for demo purposes; consider persistent storage for production)
+sessions = {}
+
+@app.post("/chat/start")
+def start_chat():
+    session_id = str(uuid4())
+    session = ChatbotSession()
+    sessions[session_id] = session
+    return {"session_id": session_id, "message": session.conversation_history[0]}
+
+@app.post("/chat/message")
+def chat_message(chat: ChatMessage):
+    if chat.session_id not in sessions:
+        raise HTTPException(status_code=404, detail="Invalid session_id")
+    
+    session = sessions[chat.session_id]
+    response = session.process_message(chat.message)
+    
+    # If the session has finished (after pain & medication), save to MongoDB and remove from memory.
+    if session.finished:
+        save_chat_session(chat.session_id, session.get_data())
+        del sessions[chat.session_id]
+    
+    return {"response": response, "conversation": session.conversation_history}
+
+@app.post("/medseg")
+async def medseg_endpoint(file: UploadFile = File(...)):
+    try:
+        contents = await file.read()
+        image = Image.open(io.BytesIO(contents)).convert("RGB")
+    except Exception:
+        raise HTTPException(status_code=400, detail="Invalid image file")
+    
+    # Process image through the complete pipeline (classification + segmentation)
+    result = complete_pipeline_image(image)
+    
+    # Save result to MongoDB
+    result_record = {
+        "filename": file.filename,
+        "result": result  # Contains predicted modality and base64 image(s)
+    }
+    save_medseg_result(result_record)
+    
+    return result

app/mediseg.py

@@ -0,0 +1,372 @@
+import os
+import torch
+import numpy as np
+import nibabel as nib
+import pydicom
+import cv2
+from PIL import Image
+import matplotlib.pyplot as plt
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision.models as models
+from torchvision import transforms
+from monai.transforms import EnsureChannelFirst, ScaleIntensity, Resize, ToTensor
+from io import BytesIO
+import base64
+
+# Set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# -------------------------------
+# CLASSIFIER MODULE (Medical Classifier)
+# -------------------------------
+class_names = ['AbdomenCT', 'BreastMRI', 'Chest Xray', 'ChestCT',
+               'Endoscopy', 'Hand Xray', 'HeadCT', 'HeadMRI']
+
+# Update model path to load from models folder
+model_path_classifier = os.path.join("models", "best_metric_model (4).pth")
+
+from monai.networks.nets import DenseNet121
+classifier_model = DenseNet121(
+    spatial_dims=2,
+    in_channels=3,
+    out_channels=len(class_names)
+).to(device)
+
+state_dict = torch.load(model_path_classifier, map_location=device)
+classifier_model.load_state_dict(state_dict, strict=False)
+classifier_model.eval()
+
+# A simple transform for classification from a PIL image
+def classify_medical_image_pil(image: Image.Image) -> str:
+    transform = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Resize((224, 224))
+    ])
+    image_tensor = transform(image).unsqueeze(0).to(device)
+    with torch.no_grad():
+        output = classifier_model(image_tensor)
+        pred_class = torch.argmax(output, dim=1).item()
+    return class_names[pred_class]
+
+# -------------------------------
+# SPECIALIZED MODULES
+# -------------------------------
+
+# --- A. Brain Tumor Segmentation Module (for HeadCT/HeadMRI) ---
+class DoubleConvUNet(nn.Module):
+    def __init__(self, in_channels, out_channels):
+        super(DoubleConvUNet, self).__init__()
+        self.conv = nn.Sequential(
+            nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU(inplace=True)
+        )
+    def forward(self, x):
+        return self.conv(x)
+
+class UNetMulti(nn.Module):
+    def __init__(self, in_channels=3, out_channels=4):
+        super(UNetMulti, self).__init__()
+        self.down1 = DoubleConvUNet(in_channels, 64)
+        self.pool1 = nn.MaxPool2d(2)
+        self.down2 = DoubleConvUNet(64, 128)
+        self.pool2 = nn.MaxPool2d(2)
+        self.down3 = DoubleConvUNet(128, 256)
+        self.pool3 = nn.MaxPool2d(2)
+        self.down4 = DoubleConvUNet(256, 512)
+        self.pool4 = nn.MaxPool2d(2)
+        self.bottleneck = DoubleConvUNet(512, 1024)
+        self.up4 = nn.ConvTranspose2d(1024, 512, kernel_size=2, stride=2)
+        self.conv4 = DoubleConvUNet(1024, 512)
+        self.up3 = nn.ConvTranspose2d(512, 256, kernel_size=2, stride=2)
+        self.conv3 = DoubleConvUNet(512, 256)
+        self.up2 = nn.ConvTranspose2d(256, 128, kernel_size=2, stride=2)
+        self.conv2 = DoubleConvUNet(256, 128)
+        self.up1 = nn.ConvTranspose2d(128, 64, kernel_size=2, stride=2)
+        self.conv1 = DoubleConvUNet(128, 64)
+        self.final_conv = nn.Conv2d(64, out_channels, kernel_size=1)
+    
+    def forward(self, x):
+        c1 = self.down1(x)
+        p1 = self.pool1(c1)
+        c2 = self.down2(p1)
+        p2 = self.pool2(c2)
+        c3 = self.down3(p2)
+        p3 = self.pool3(c3)
+        c4 = self.down4(p3)
+        p4 = self.pool4(c4)
+        bn = self.bottleneck(p4)
+        u4 = self.up4(bn)
+        merge4 = torch.cat([u4, c4], dim=1)
+        c5 = self.conv4(merge4)
+        u3 = self.up3(c5)
+        merge3 = torch.cat([u3, c3], dim=1)
+        c6 = self.conv3(merge3)
+        u2 = self.up2(c6)
+        merge2 = torch.cat([u2, c2], dim=1)
+        c7 = self.conv2(merge2)
+        u1 = self.up1(c7)
+        merge1 = torch.cat([u1, c1], dim=1)
+        c8 = self.conv1(merge1)
+        output = self.final_conv(c8)
+        return output
+
+def process_brain_tumor(image: Image.Image, model_path=os.path.join("models", "brain_tumor_unet_multiclass.pth")) -> str:
+    model = UNetMulti(in_channels=3, out_channels=4).to(device)
+    model.load_state_dict(torch.load(model_path, map_location=device))
+    model.eval()
+    
+    transform_img = transforms.Compose([
+        transforms.Resize((256,256)),
+        transforms.ToTensor()
+    ])
+    input_tensor = transform_img(image).unsqueeze(0).to(device)
+    with torch.no_grad():
+        output = model(input_tensor)
+        preds = torch.argmax(output, dim=1).squeeze().cpu().numpy()
+    
+    image_np = np.array(image.resize((256,256)))
+    # Create overlay and blended image
+    overlay = cv2.applyColorMap(np.uint8(255 * preds/np.max(preds + 1e-8)), cv2.COLORMAP_JET)
+    overlay = cv2.cvtColor(overlay, cv2.COLOR_BGR2RGB)
+    blended = cv2.addWeighted(np.uint8(image_np), 0.6, overlay, 0.4, 0)
+    
+    # Create a figure with subplots
+    fig, ax = plt.subplots(1, 3, figsize=(18,6))
+    ax[0].imshow(image_np)
+    ax[0].set_title("Original Image")
+    ax[0].axis("off")
+    ax[1].imshow(preds, cmap='jet')
+    ax[1].set_title("Segmentation Mask")
+    ax[1].axis("off")
+    ax[2].imshow(blended)
+    ax[2].set_title("Overlay")
+    ax[2].axis("off")
+    
+    buf = BytesIO()
+    fig.savefig(buf, format="png")
+    buf.seek(0)
+    img_base64 = base64.b64encode(buf.read()).decode("utf-8")
+    plt.close(fig)
+    return img_base64
+
+# --- B. Endoscopy Polyp Detection Module (Binary UNet) ---
+class UNetBinary(nn.Module):
+    def __init__(self, in_channels=3, out_channels=1):
+        super(UNetBinary, self).__init__()
+        self.down1 = DoubleConvUNet(in_channels, 64)
+        self.pool1 = nn.MaxPool2d(2)
+        self.down2 = DoubleConvUNet(64, 128)
+        self.pool2 = nn.MaxPool2d(2)
+        self.down3 = DoubleConvUNet(128, 256)
+        self.pool3 = nn.MaxPool2d(2)
+        self.down4 = DoubleConvUNet(128, 512)
+        self.pool4 = nn.MaxPool2d(2)
+        self.bottleneck = DoubleConvUNet(512, 1024)
+        self.up4 = nn.ConvTranspose2d(1024, 512, kernel_size=2, stride=2)
+        self.conv4 = DoubleConvUNet(1024, 512)
+        self.up3 = nn.ConvTranspose2d(512, 256, kernel_size=2, stride=2)
+        self.conv3 = DoubleConvUNet(512, 256)
+        self.up2 = nn.ConvTranspose2d(256, 128, kernel_size=2, stride=2)
+        self.conv2 = DoubleConvUNet(256, 128)
+        self.up1 = nn.ConvTranspose2d(128, 64, kernel_size=2, stride=2)
+        self.conv1 = DoubleConvUNet(128, 64)
+        self.final_conv = nn.Conv2d(64, out_channels, kernel_size=1)
+    
+    def forward(self, x):
+        c1 = self.down1(x)
+        p1 = self.pool1(c1)
+        c2 = self.down2(p1)
+        p2 = self.pool2(c2)
+        c3 = self.down3(p2)
+        p3 = self.pool3(c3)
+        c4 = self.down4(p3)
+        p4 = self.pool4(c4)
+        bn = self.bottleneck(p4)
+        u4 = self.up4(bn)
+        merge4 = torch.cat([u4, c4], dim=1)
+        c5 = self.conv4(merge4)
+        u3 = self.up3(c5)
+        merge3 = torch.cat([u3, c3], dim=1)
+        c6 = self.conv3(merge3)
+        u2 = self.up2(c6)
+        merge2 = torch.cat([u2, c2], dim=1)
+        c7 = self.conv2(merge2)
+        u1 = self.up1(c7)
+        merge1 = torch.cat([u1, c1], dim=1)
+        c8 = self.conv1(merge1)
+        output = self.final_conv(c8)
+        return output
+
+def process_endoscopy(image: Image.Image, model_path=os.path.join("models", "endoscopy_unet.pth")) -> str:
+    model = UNetBinary(in_channels=3, out_channels=1).to(device)
+    model.load_state_dict(torch.load(model_path, map_location=device))
+    model.eval()
+    
+    transform_img = transforms.Compose([
+        transforms.Resize((256,256)),
+        transforms.ToTensor()
+    ])
+    input_tensor = transform_img(image).unsqueeze(0).to(device)
+    with torch.no_grad():
+        output = model(input_tensor)
+        prob = torch.sigmoid(output)
+        mask = (prob > 0.5).float().squeeze().cpu().numpy()
+    
+    image_np = np.array(image.resize((256,256)))
+    overlay = cv2.applyColorMap(np.uint8(255*mask), cv2.COLORMAP_JET)
+    overlay = cv2.cvtColor(overlay, cv2.COLOR_BGR2RGB)
+    blended = cv2.addWeighted(np.uint8(image_np), 0.6, overlay, 0.4, 0)
+    
+    fig, ax = plt.subplots(1, 3, figsize=(18,6))
+    ax[0].imshow(image_np)
+    ax[0].set_title("Actual Image")
+    ax[0].axis("off")
+    ax[1].imshow(mask, cmap='gray')
+    ax[1].set_title("Segmentation Mask")
+    ax[1].axis("off")
+    ax[2].imshow(blended)
+    ax[2].set_title("Overlay")
+    ax[2].axis("off")
+    
+    buf = BytesIO()
+    fig.savefig(buf, format="png")
+    buf.seek(0)
+    img_base64 = base64.b64encode(buf.read()).decode("utf-8")
+    plt.close(fig)
+    return img_base64
+
+# --- C. Pneumonia Detection Module (Using Grad-CAM on ResNet18) ---
+class GradCAM_Pneumonia:
+    def __init__(self, model, target_layer):
+        self.model = model
+        self.target_layer = target_layer
+        self.gradients = None
+        self.activations = None
+        self.hook_handles = []
+        self._register_hooks()
+    
+    def _register_hooks(self):
+        def forward_hook(module, input, output):
+            self.activations = output.detach()
+        def backward_hook(module, grad_in, grad_out):
+            self.gradients = grad_out[0].detach()
+        handle1 = self.target_layer.register_forward_hook(forward_hook)
+        handle2 = self.target_layer.register_backward_hook(backward_hook)
+        self.hook_handles.extend([handle1, handle2])
+    
+    def remove_hooks(self):
+        for handle in self.hook_handles:
+            handle.remove()
+    
+    def generate(self, input_image, target_class=None):
+        output = self.model(input_image)
+        if target_class is None:
+            target_class = output.argmax(dim=1).item()
+        self.model.zero_grad()
+        one_hot = torch.zeros_like(output)
+        one_hot[0, target_class] = 1
+        output.backward(gradient=one_hot, retain_graph=True)
+        weights = self.gradients.mean(dim=(2,3), keepdim=True)
+        cam = (weights * self.activations).sum(dim=1, keepdim=True)
+        cam = F.relu(cam)
+        cam = cam.squeeze().cpu().numpy()
+        _, _, H, W = input_image.shape
+        cam = cv2.resize(cam, (W, H))
+        cam = (cam - np.min(cam)) / (np.max(cam) - np.min(cam) + 1e-8)
+        return cam, output
+
+def process_pneumonia(image: Image.Image, model_path=os.path.join("models", "pneumonia_resnet18.pth")) -> str:
+    model = models.resnet18(pretrained=False)
+    num_ftrs = model.fc.in_features
+    model.fc = nn.Linear(num_ftrs, 2)  # 2 classes: normal and pneumonia
+    model.load_state_dict(torch.load(model_path, map_location=device))
+    model.to(device)
+    model.eval()
+    
+    grad_cam = GradCAM_Pneumonia(model, model.layer4)
+    
+    transform_img = transforms.Compose([
+        transforms.Resize((224,224)),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.485,0.456,0.406],
+                             std=[0.229,0.224,0.225])
+    ])
+    input_tensor = transform_img(image).unsqueeze(0).to(device)
+    with torch.no_grad():
+        cam, output = grad_cam.generate(input_tensor)
+        predicted_class = output.argmax(dim=1).item()
+    label_text = "Pneumonia" if predicted_class == 1 else "Normal"
+    
+    def get_bounding_box(heatmap, thresh=0.5, min_area=100):
+        heat_uint8 = np.uint8(255 * heatmap)
+        ret, binary = cv2.threshold(heat_uint8, int(thresh*255), 255, cv2.THRESH_BINARY)
+        contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+        if len(contours)==0:
+            return None
+        largest = max(contours, key=cv2.contourArea)
+        if cv2.contourArea(largest) < min_area:
+            return None
+        x, y, w, h = cv2.boundingRect(largest)
+        return (x, y, w, h)
+    
+    bbox = None
+    if predicted_class == 1:
+        bbox = get_bounding_box(cam, thresh=0.5, min_area=100)
+    
+    resized_image = image.resize((224,224))
+    image_np = np.array(resized_image)
+    overlay = image_np.copy()
+    if bbox is not None:
+        x, y, w, h = bbox
+        cv2.rectangle(overlay, (x, y), (x+w, y+h), (255,0,0), 2)
+    cv2.putText(overlay, label_text, (10,25), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255,255,0),2)
+    
+    heatmap_color = cv2.applyColorMap(np.uint8(255*cam), cv2.COLORMAP_JET)
+    heatmap_color = cv2.cvtColor(heatmap_color, cv2.COLOR_BGR2RGB)
+    
+    fig, ax = plt.subplots(1, 3, figsize=(18,6))
+    ax[0].imshow(image_np)
+    ax[0].set_title("Actual Image")
+    ax[0].axis("off")
+    ax[1].imshow(heatmap_color)
+    ax[1].set_title("Detected Output (Heatmap)")
+    ax[1].axis("off")
+    ax[2].imshow(overlay)
+    ax[2].set_title("Boxed Overlay")
+    ax[2].axis("off")
+    
+    buf = BytesIO()
+    fig.savefig(buf, format="png")
+    buf.seek(0)
+    img_base64 = base64.b64encode(buf.read()).decode("utf-8")
+    plt.close(fig)
+    grad_cam.remove_hooks()
+    return img_base64
+
+# -------------------------------
+# COMPLETE PIPELINE FUNCTION
+# -------------------------------
+def complete_pipeline_image(image: Image.Image) -> dict:
+    predicted_modality = classify_medical_image_pil(image)
+    result = {"predicted_modality": predicted_modality}
+    
+    if predicted_modality in ["HeadCT", "HeadMRI"]:
+        result_overlay = process_brain_tumor(image)
+        result["segmentation_result"] = result_overlay
+    elif predicted_modality == "Endoscopy":
+        result_overlay = process_endoscopy(image)
+        result["segmentation_result"] = result_overlay
+    elif predicted_modality == "Chest Xray":
+        result_overlay = process_pneumonia(image)
+        result["segmentation_result"] = result_overlay
+    else:
+        # For modalities without specialized processing, return the original image as base64
+        buf = BytesIO()
+        image.save(buf, format="PNG")
+        result["segmentation_result"] = base64.b64encode(buf.getvalue()).decode("utf-8")
+    return result

models/best_metric_model (4).pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a5e578d9e93b089eed0bdbdaa50237209fce830497136593664b16d4df720ee
+size 28471314

models/brain_tumor_unet_multiclass.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b11bc3ebaa317154e9530f8151ce6bf7efa407abeedc62454502099889dbfe42
+size 124269778

models/endoscopy_unet.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cac633092591e1c515da87ed7bd54de35ca4c50fec41a646b8cfef5b1e15afde
+size 124267126

models/pneumonia_resnet18.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a9bca18323c658623f0e62207e6c3331836ad1825a8f9c9d3aa118709d2614a
+size 44790376

requirements.txt

@@ -0,0 +1,16 @@
+fastapi
+uvicorn[standard]
+pymongo
+pandas
+faiss-cpu
+numpy
+scikit-learn
+transformers
+torch
+torchvision
+monai
+pydicom
+nibabel
+opencv-python
+Pillow
+matplotlib

symptom_assessment.py

@@ -0,0 +1,31 @@
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+class SymptomAssessment:
+    def __init__(self):
+        # Example disease-symptom mapping dictionary.
+        # In practice, replace this with a robust dataset.
+        self.disease_symptoms = {
+            "Flu": ["fever", "cough", "sore throat", "fatigue"],
+            "Migraine": ["headache", "nausea", "sensitivity to light"],
+            "COVID-19": ["fever", "cough", "shortness of breath", "loss of taste"]
+        }
+        # Prepare vector space for diseases
+        self.vectorizer = TfidfVectorizer()
+        self.diseases = list(self.disease_symptoms.keys())
+        symptom_texts = [" ".join(self.disease_symptoms[d]) for d in self.diseases]
+        self.vectors = self.vectorizer.fit_transform(symptom_texts)
+    
+    def assess(self, symptoms_list):
+        """
+        Given a list of reported symptoms, determine the best matching disease
+        and identify which expected symptoms are missing.
+        """
+        input_text = " ".join(symptoms_list)
+        input_vector = self.vectorizer.transform([input_text])
+        similarities = (self.vectors * input_vector.T).toarray().flatten()
+        best_match_index = similarities.argmax()
+        best_disease = self.diseases[best_match_index]
+        missing_symptoms = list(set(self.disease_symptoms[best_disease]) - set(symptoms_list))
+        assessment = (f"Based on the input symptoms, {best_disease} is suspected. "
+                      f"Missing symptoms for improved diagnosis: {missing_symptoms}")
+        return missing_symptoms, assessment

templates/index.html

@@ -0,0 +1,51 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>Medical Image Processing Pipeline</title>
+  <style>
+    body { font-family: Arial, sans-serif; margin: 20px; }
+    .container { max-width: 800px; margin: auto; }
+    .result img { max-width: 250px; margin: 10px; }
+    .result { display: flex; flex-wrap: wrap; }
+  </style>
+</head>
+<body>
+  <div class="container">
+    <h1>Medical Image Processing Pipeline</h1>
+    <form action="/predict" method="POST" enctype="multipart/form-data">
+      <input type="file" name="file" accept="image/*,.nii,.nii.gz,.dcm" required>
+      <button type="submit">Upload and Process</button>
+    </form>
+    {% if result %}
+      <hr>
+      {% if result.error %}
+        <h3>Error: {{ result.error }}</h3>
+      {% else %}
+        <h3>Predicted Modality: {{ result.predicted_modality }}</h3>
+        {% if result.specialized %}
+          <div class="result">
+            <div>
+              <h4>Original Image</h4>
+              <img src="data:image/jpeg;base64,{{ result.specialized.original }}" alt="Original">
+            </div>
+            <div>
+              <h4>Mask</h4>
+              <img src="data:image/jpeg;base64,{{ result.specialized.mask }}" alt="Mask">
+            </div>
+            <div>
+              <h4>Overlay</h4>
+              <img src="data:image/jpeg;base64,{{ result.specialized.overlay }}" alt="Overlay">
+            </div>
+          </div>
+        {% elif result.message %}
+          <h4>{{ result.message }}</h4>
+          <div class="result">
+            <img src="data:image/jpeg;base64,{{ result.original }}" alt="Original">
+          </div>
+        {% endif %}
+      {% endif %}
+    {% endif %}
+  </div>
+</body>
+</html>