added_all

Dockerfile

@@ -0,0 +1,16 @@
+# Use an official Python runtime as a base image
+FROM python:3.9-slim
+
+# Set the working directory
+WORKDIR /app
+
+# Copy the current directory contents into the container at /app
+COPY . /app
+
+# Install the dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Expose port 8000
+EXPOSE 7860
+# CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

Phronesis

@@ -1 +0,0 @@
-Subproject commit 9c5facf38621ad02be7be79226d794f6c2f14dee

README.md

@@ -0,0 +1,13 @@
+---
+title: Phronesis
+emoji: 🌖
+colorFrom: green
+colorTo: gray
+sdk: gradio
+sdk_version: 5.4.0
+app_file: app.py
+pinned: false
+short_description: 'REPORT GEN AND CLASSIFICATION MODEL '
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,121 @@
+#app.py
+import os
+import io
+import uvicorn
+
+import torch
+from fastapi import FastAPI, File, UploadFile, HTTPException
+from fastapi.responses import JSONResponse
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+from torchvision import models, transforms
+from PIL import Image
+import numpy as np
+from huggingface_hub import hf_hub_download
+import pydicom
+import gc
+from model import CombinedModel, ImageToTextProjector
+
+from fastapi import FastAPI, Request
+
+app = FastAPI()
+
+@app.get("/")
+async def root(request: Request):
+    return {"message": "Welcome to Phronesis"}
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+def dicom_to_png(dicom_data):
+    try:
+        dicom_file = pydicom.dcmread(dicom_data)
+        if not hasattr(dicom_file, 'PixelData'):
+            raise HTTPException(status_code=400, detail="No pixel data in DICOM file.")
+        
+        pixel_array = dicom_file.pixel_array.astype(np.float32)
+        pixel_array = ((pixel_array - pixel_array.min()) / (pixel_array.ptp())) * 255.0
+        pixel_array = pixel_array.astype(np.uint8)
+
+        img = Image.fromarray(pixel_array).convert("L")
+        return img
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error converting DICOM to PNG: {e}")
+
+# Set up secure model initialization
+HF_TOKEN = os.getenv('HF_TOKEN')
+if not HF_TOKEN:
+    raise ValueError("Missing Hugging Face token in environment variables.")
+
+try:
+    report_generator_tokenizer = AutoTokenizer.from_pretrained(
+        "KYAGABA/combined-multimodal-model",
+        token=HF_TOKEN if HF_TOKEN else None
+    )
+    video_model = models.video.r3d_18(weights="KINETICS400_V1")
+    video_model.fc = torch.nn.Linear(video_model.fc.in_features, 512)
+    report_generator = AutoModelForSeq2SeqLM.from_pretrained("GanjinZero/biobart-v2-base")
+    projector = ImageToTextProjector(512, report_generator.config.d_model)
+    num_classes = 4
+    combined_model = CombinedModel(video_model, report_generator, num_classes, projector, report_generator_tokenizer)
+    model_file = hf_hub_download("KYAGABA/combined-multimodal-model", "pytorch_model.bin", token=HF_TOKEN)
+    state_dict = torch.load(model_file, map_location=device)
+    combined_model.load_state_dict(state_dict)
+    combined_model.eval()
+except Exception as e:
+    raise SystemExit(f"Error loading models: {e}")
+
+image_transform = transforms.Compose([
+    transforms.Resize((112, 112)),
+    transforms.ToTensor(),
+    transforms.Normalize(mean=[0.43216, 0.394666, 0.37645], std=[0.22803, 0.22145, 0.216989])
+])
+
+class_names = ["acute", "normal", "chronic", "lacunar"]
+
+@app.post("/predict/")
+async def predict(files: list[UploadFile]):
+    print(f"Received {len(files)} files")
+    n_frames = 16
+    images = []
+
+    for file in files:
+        ext = file.filename.split('.')[-1].lower()
+        try:
+            if ext in ['dcm', 'ima']:
+                dicom_img = dicom_to_png(await file.read())
+                images.append(dicom_img.convert("RGB"))
+            elif ext in ['png', 'jpeg', 'jpg']:
+                img = Image.open(io.BytesIO(await file.read())).convert("RGB")
+                images.append(img)
+            else:
+                raise HTTPException(status_code=400, detail="Unsupported file type.")
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Error processing file {file.filename}: {e}")
+
+    if not images:
+        return JSONResponse(content={"error": "No valid images provided."}, status_code=400)
+
+    if len(images) >= n_frames:
+        images_sampled = [images[i] for i in np.linspace(0, len(images) - 1, n_frames, dtype=int)]
+    else:
+        images_sampled = images + [images[-1]] * (n_frames - len(images))
+
+    image_tensors = [image_transform(img) for img in images_sampled]
+    images_tensor = torch.stack(image_tensors).permute(1, 0, 2, 3).unsqueeze(0).to(device)
+
+    with torch.no_grad():
+        class_outputs, generated_report, _ = combined_model(images_tensor)
+        predicted_class = torch.argmax(class_outputs, dim=1).item()
+        predicted_class_name = class_names[predicted_class]
+
+    gc.collect()
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+
+    return {
+        "predicted_class": predicted_class_name,
+        "generated_report": generated_report[0] if generated_report else "No report generated."
+    }
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", 7860)))

model.py

@@ -0,0 +1,56 @@
+# model.py
+
+import torch
+import torch.nn as nn
+from transformers import AutoModelForSeq2SeqLM
+
+class ImageToTextProjector(nn.Module):
+    def __init__(self, image_embedding_dim, text_embedding_dim):
+        super(ImageToTextProjector, self).__init__()
+        self.fc = nn.Linear(image_embedding_dim, text_embedding_dim)
+        self.activation = nn.ReLU()
+        self.dropout = nn.Dropout(p=0.5)
+
+    def forward(self, x):
+        x = self.fc(x)
+        x = self.activation(x)
+        x = self.dropout(x)
+        return x
+
+class CombinedModel(nn.Module):
+    def __init__(self, video_model, report_generator, num_classes, projector, tokenizer):
+        super(CombinedModel, self).__init__()
+        self.video_model = video_model
+        self.report_generator = report_generator
+        self.classifier = nn.Linear(512, num_classes)
+        self.projector = projector
+        self.dropout = nn.Dropout(p=0.5)
+        self.tokenizer = tokenizer  # Store tokenizer
+
+    def forward(self, images, labels=None):
+        video_embeddings = self.video_model(images)
+        video_embeddings = self.dropout(video_embeddings)
+        class_outputs = self.classifier(video_embeddings)
+        projected_embeddings = self.projector(video_embeddings)
+        encoder_inputs = projected_embeddings.unsqueeze(1)
+
+        if labels is not None:
+            outputs = self.report_generator(
+                inputs_embeds=encoder_inputs,
+                labels=labels
+            )
+            gen_loss = outputs.loss
+            generated_report = None
+        else:
+            generated_report_ids = self.report_generator.generate(
+                inputs_embeds=encoder_inputs,
+                max_length=512,
+                num_beams=4,
+                early_stopping=True
+            )
+            generated_report = self.tokenizer.batch_decode(
+                generated_report_ids, skip_special_tokens=True
+            )
+            gen_loss = None
+
+        return class_outputs, generated_report, gen_loss

requirements.txt

@@ -0,0 +1,16 @@
+# Core dependencies
+torch==2.0.1
+torchvision==0.15.2
+transformers==4.44.2
+gradio==5.0
+numpy==1.26.2
+Pillow==10.0.1
+fastapi
+# Additional dependencies
+huggingface_hub==0.25.1  # Compatible with both transformers and gradio
+torchmetrics==1.5.1
+nltk==3.8.1
+scikit-learn==1.3.0
+tqdm==4.66.1
+sentencepiece==0.1.99
+pydicom==2.4.1