Upload 4 files

README.md

@@ -0,0 +1,87 @@
+# Clothes Segmentation
+
+![Sample Images and Segmentation Masks from Dataset](assets/dataset_examples.png)
+
+This project provides a solution for segmenting clothes into 18 categories using DINO, ViT and UNet models.
+
+* DINO: Pretrained backbone with a segmentation head
+  * https://arxiv.org/abs/2104.14294
+  * https://huggingface.co/facebook/dinov2-small
+* ViT: Pretrained vision transformer with a segmentation head
+  * https://arxiv.org/abs/2010.11929
+  * https://huggingface.co/google/vit-base-patch16-224
+* UNet: Custom implementation
+  * https://arxiv.org/abs/1505.04597
+
+Gradio is used for building a web interface and Weights & Biases for experiments tracking.
+
+## Installation
+
+1. Clone the repository:
+    ```bash
+    git clone https://github.com/your-project/clothes-segmentation.git
+    cd plant-classifier
+    ```
+
+2. Create and activate a virtual environment:
+    ```bash
+    python -m venv venv
+    source venv/bin/activate
+    ```
+
+3. Install dependencies:
+    ```bash
+    pip install -r requirements.txt
+    ```
+
+## Usage
+
+### Training the Model
+   To train a model, specify one of the following using the --model argument: **dino**, **vit** or **unet**.
+   ```bash
+   python src/train.py --model dino
+   python src/train.py --model vit
+   python src/train.py --model unet
+   ```
+
+   You can also adjust other parameters, such as the number of epochs, batch size, and learning rate, by adding additional arguments. For example:
+   ```bash
+   python src/train.py --model unet --num-epochs 20 --batch-size 16 --learning-rate 0.001
+   ```
+
+### Launching the Gradio Interface
+   ```bash
+   python app.py
+   ```
+
+   Once the interface is running, you can select a model, upload an image and view the segmentation mask.
+   
+   ![Web Interface Screen](assets/spaces_screen.jpg)
+
+   #### добавить ссылку
+
+## Results
+
+| Model      | Test Micro Recall | Test Micro Precision | Test Macro Precision | Test Macro Recall | Test Accuracy | Test Loss | Train Micro Recall | Train Micro Precision | Train Macro Precision | Train Macro Recall | Train Accuracy | Train Loss |
+|------------|-------------------|----------------------|----------------------|-------------------|---------------|-----------|--------------------|-----------------------|-----------------------|--------------------|----------------|------------|
+| DINO       | 0.94986          | 0.94986             | 0.71364             | 0.67052          | 0.94986       | 0.53124   | 0.97019            | 0.97019              | 0.78185              | 0.72336           | 0.97019        | 0.30441    |
+| ViT        | 0.9358           | 0.9358              | 0.63939             | 0.58365          | 0.9358        | 0.71193   | 0.96734            | 0.96734              | 0.74418              | 0.66295           | 0.96734        | 0.31166    |
+| UNet       | 0.95798          | 0.95798             | 0.76354             | 0.7289           | 0.95798       | 0.56764   | 0.98035            | 0.98035              | 0.82934              | 0.82688           | 0.98035        | 0.25301    |
+
+### Training Results of DINO
+
+![DINO_test](assets/dino_test_plots.png)
+
+![DINO_train](assets/dino_train_plots.png)
+
+### Training Results of ViT
+
+![ViT_test](assets/vit_test_plots.png)
+
+![ViT_train](assets/vit_train_plots.png)
+
+### Training Results of UNet
+
+![UNet_test](assets/unet_test_plots.png)
+
+![UNet_train](assets/unet_train_plots.png)

app.py

@@ -0,0 +1,128 @@
+from typing import Dict
+import gradio as gr
+import json
+import PIL.Image, PIL.ImageOps
+import torch
+import torchvision.transforms.functional as F
+from matplotlib import cm
+from matplotlib.colors import to_hex
+import numpy as np
+
+from src.models.dino import DINOSegmentationModel
+from src.models.vit import ViTSegmentation
+from src.models.unet import UNet
+from src.utils import get_transform
+
+
+device = torch.device("cpu")
+model_weight1 = "weights/dino.pth"
+model_weight2 = "weights/vit.pth"
+model_weight3 = "weights/unet.pth"
+
+model1 = DINOSegmentationModel()
+model1.segmentation_head.load_state_dict(torch.load(model_weight1, map_location=device))
+model1.eval()
+model2 = ViTSegmentation()
+model2.segmentation_head.load_state_dict(torch.load(model_weight2, map_location=device))
+model2.eval()
+model3 = UNet()
+model3.load_state_dict(torch.load(model_weight3, map_location=device))
+model3.eval()
+
+mask_labels = {
+    "0": "Background", "1": "Hat", "2": "Hair", "3": "Sunglasses", "4": "Upper-clothes",
+    "5": "Skirt", "6": "Pants", "7": "Dress", "8": "Belt", "9": "Right-shoe",
+    "10": "Left-shoe", "11": "Face", "12": "Right-leg", "13": "Left-leg", 
+    "14": "Right-arm", "15": "Left-arm", "16": "Bag", "17": "Scarf"
+}
+
+color_map = cm.get_cmap('tab20', 18)
+label_colors = {label: to_hex(color_map(idx / len(mask_labels))[:3]) for idx, label in enumerate(mask_labels)}
+fixed_colors = np.array([color_map(i)[:3] for i in range(18)]) * 255
+
+
+def mask_to_color(mask: np.ndarray) -> np.ndarray:
+    h, w = mask.shape
+    color_mask = np.zeros((h, w, 3), dtype=np.uint8)
+    for class_idx in range(18):
+        color_mask[mask == class_idx] = fixed_colors[class_idx]
+    return color_mask
+
+
+def segment_image(image, model_name: str) -> PIL.Image:
+    if model_name == "DINO":
+        model = model1
+    elif model_name == "ViT":
+        model = model2
+    else:
+        model = model3
+
+    original_width, original_height = image.size
+    transform = get_transform(model.mean, model.std)
+    input_tensor = transform(image).unsqueeze(0)
+
+    with torch.no_grad():
+        mask = model(input_tensor)
+    mask = torch.argmax(mask.squeeze(), dim=0).cpu().numpy()
+
+    mask_image = mask_to_color(mask)
+
+    mask_image = PIL.Image.fromarray(mask_image)
+    mask_aspect_ratio = mask_image.width / mask_image.height
+
+    new_height = original_height
+    new_width = int(new_height * mask_aspect_ratio)
+    mask_image = mask_image.resize((new_width, new_height), PIL.Image.Resampling.NEAREST)
+
+    final_mask = PIL.Image.new("RGB", (original_width, original_height))
+    offset = ((original_width - new_width) // 2, 0)
+    final_mask.paste(mask_image, offset)
+
+    return final_mask
+
+
+def generate_legend_html_compact() -> str:
+    legend_html = """
+    <div style='display: flex; flex-wrap: wrap; gap: 10px; justify-content: center;'>
+    """
+    for idx, (label, color) in enumerate(label_colors.items()):
+        legend_html += f"""
+        <div style='display: flex; align-items: center; justify-content: center; 
+                     padding: 5px 10px; border: 1px solid {color}; 
+                     background-color: {color}; border-radius: 5px; 
+                     color: white; font-size: 12px; text-align: center;'>
+            {mask_labels[label]}
+        </div>
+        """
+    legend_html += "</div>"
+    return legend_html
+
+
+examples = [
+    ["assets/images_examples/image1.jpg"],
+    ["assets/images_examples/image2.jpg"],
+    ["assets/images_examples/image3.jpg"]
+]
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("## Clothes Segmentation")
+    with gr.Row():
+        with gr.Column():
+            pic = gr.Image(label="Upload Human Image", type="pil", height=300, width=300)
+            model_choice = gr.Dropdown(choices=["DINO", "ViT", "UNet"], label="Select Model", value="DINO")
+            with gr.Row():
+                with gr.Column(scale=1):
+                    predict_btn = gr.Button("Predict")
+                with gr.Column(scale=1):
+                    clear_btn = gr.Button("Clear")
+        
+        with gr.Column():
+            output = gr.Image(label="Mask", type="pil", height=300, width=300)
+            legend = gr.HTML(label="Legend", value=generate_legend_html_compact())
+
+    predict_btn.click(fn=segment_image, inputs=[pic, model_choice], outputs=output, api_name="predict")
+    clear_btn.click(lambda: (None, None), outputs=[pic, output])
+    gr.Examples(examples=examples, inputs=[pic])
+
+demo.launch()

losses_config.json

@@ -0,0 +1,14 @@
+{
+    "cross_entropy": 0.33,
+    "SSLoss_v2": 0.0,
+    "ExpLog_loss": 0.0,
+    "LovaszSoftmax": 0.0,
+    "TopKLoss": 0.33,
+    "WeightedCrossEntropyLoss": 0.0,
+    "SoftDiceLoss_v2": 0.0,
+    "IoULoss_v2": 0.0,
+    "TverskyLoss_v2": 0.0,
+    "FocalTversky_loss_v2": 0.0,
+    "AsymLoss_v2": 0.0,
+    "FocalLoss": 0.33
+}

requirements.txt

@@ -0,0 +1,11 @@
+torch==2.4.1
+torchvision==0.19.1
+kaggle==1.6.17
+wandb==0.18.5
+gradio==5.4.0
+datasets==3.1.0
+accelerate==1.1.0
+opencv-python==4.10.0.84
+scipy==1.14.1
+transformers==4.46.2
+matplotlib==3.10.0