🔥 [Remove] utils and config again, move to yolo/

config/config.py

@@ -1,72 +0,0 @@
-from dataclasses import dataclass
-from typing import Dict, List, Union
-
-
-@dataclass
-class Model:
-    anchor: List[List[int]]
-    model: Dict[str, List[Dict[str, Union[Dict, List, int]]]]
-
-
-@dataclass
-class Download:
-    auto: bool
-    path: str
-
-
-@dataclass
-class DataLoaderConfig:
-    batch_size: int
-    shuffle: bool
-    num_workers: int
-    pin_memory: bool
-
-
-@dataclass
-class OptimizerArgs:
-    lr: float
-    weight_decay: float
-
-
-@dataclass
-class OptimizerConfig:
-    type: str
-    args: OptimizerArgs
-
-
-@dataclass
-class SchedulerArgs:
-    step_size: int
-    gamma: float
-
-
-@dataclass
-class SchedulerConfig:
-    type: str
-    args: SchedulerArgs
-
-
-@dataclass
-class EMAConfig:
-    enabled: bool
-    decay: float
-
-
-@dataclass
-class TrainConfig:
-    optimizer: OptimizerConfig
-    scheduler: SchedulerConfig
-    ema: EMAConfig
-
-
-@dataclass
-class HyperConfig:
-    data: DataLoaderConfig
-    train: TrainConfig
-
-
-@dataclass
-class Config:
-    model: Model
-    download: Download
-    hyper: HyperConfig

config/config.yaml

@@ -1,11 +0,0 @@
-hydra:
-  run:
-    dir: ./runs
-
-defaults:
- - data: coco
- - download: ../data/download
- - augmentation: ../data/augmentation
- - model: v7-base
- - hyper: default
- - _self_

config/data/augmentation.yaml

@@ -1,3 +0,0 @@
-Mosaic: 1
-# MixUp: 1
-HorizontalFlip: 0.5

config/data/download.yaml

@@ -1,17 +0,0 @@
-auto: True
-path: data/coco
-images:
-  base_url: http://images.cocodataset.org/zips/
-  datasets:
-    train:
-      file_name: train2017.zip
-      file_num: 118287
-    val: 
-      file_name: val2017.zip
-      num_files: 5000
-    test:
-      file_name: test2017.zip
-      num_files: 40670
-hydra:
-  run:
-    dir: ./runs

config/hyper/default.yaml

@@ -1,19 +0,0 @@
-data:
-  batch_size: 4 
-  shuffle: True
-  num_workers: 4
-  pin_memory: True
-train:
-  optimizer:
-    type: Adam
-    args:
-      lr: 0.001
-      weight_decay: 0.0001
-  scheduler:
-    type: StepLR
-    args:
-      step_size: 10
-      gamma: 0.1
-  ema:
-      enabled: true
-      decay: 0.995

utils/data_augment.py

@@ -1,125 +0,0 @@
-import numpy as np
-import torch
-from PIL import Image
-from torchvision.transforms import functional as TF
-
-
-class Compose:
-    """Composes several transforms together."""
-
-    def __init__(self, transforms, image_size: int = 640):
-        self.transforms = transforms
-        self.image_size = image_size
-
-        for transform in self.transforms:
-            if hasattr(transform, "set_parent"):
-                transform.set_parent(self)
-
-    def __call__(self, image, boxes):
-        for transform in self.transforms:
-            image, boxes = transform(image, boxes)
-        return image, boxes
-
-
-class HorizontalFlip:
-    """Randomly horizontally flips the image along with the bounding boxes."""
-
-    def __init__(self, prob=0.5):
-        self.prob = prob
-
-    def __call__(self, image, boxes):
-        if torch.rand(1) < self.prob:
-            image = TF.hflip(image)
-            boxes[:, [1, 3]] = 1 - boxes[:, [3, 1]]
-        return image, boxes
-
-
-class VerticalFlip:
-    """Randomly vertically flips the image along with the bounding boxes."""
-
-    def __init__(self, prob=0.5):
-        self.prob = prob
-
-    def __call__(self, image, boxes):
-        if torch.rand(1) < self.prob:
-            image = TF.vflip(image)
-            boxes[:, [2, 4]] = 1 - boxes[:, [4, 2]]
-        return image, boxes
-
-
-class Mosaic:
-    """Applies the Mosaic augmentation to a batch of images and their corresponding boxes."""
-
-    def __init__(self, prob=0.5):
-        self.prob = prob
-        self.parent = None
-
-    def set_parent(self, parent):
-        self.parent = parent
-
-    def __call__(self, image, boxes):
-        if torch.rand(1) >= self.prob:
-            return image, boxes
-
-        assert self.parent is not None, "Parent is not set. Mosaic cannot retrieve image size."
-
-        img_sz = self.parent.image_size  # Assuming `image_size` is defined in parent
-        more_data = self.parent.get_more_data(3)  # get 3 more images randomly
-
-        data = [(image, boxes)] + more_data
-        mosaic_image = Image.new("RGB", (2 * img_sz, 2 * img_sz))
-        vectors = np.array([(-1, -1), (0, -1), (-1, 0), (0, 0)])
-        center = np.array([img_sz, img_sz])
-        all_labels = []
-
-        for (image, boxes), vector in zip(data, vectors):
-            this_w, this_h = image.size
-            coord = tuple(center + vector * np.array([this_w, this_h]))
-
-            mosaic_image.paste(image, coord)
-            xmin, ymin, xmax, ymax = boxes[:, 1], boxes[:, 2], boxes[:, 3], boxes[:, 4]
-            xmin = (xmin * this_w + coord[0]) / (2 * img_sz)
-            xmax = (xmax * this_w + coord[0]) / (2 * img_sz)
-            ymin = (ymin * this_h + coord[1]) / (2 * img_sz)
-            ymax = (ymax * this_h + coord[1]) / (2 * img_sz)
-
-            adjusted_boxes = torch.stack([boxes[:, 0], xmin, ymin, xmax, ymax], dim=1)
-            all_labels.append(adjusted_boxes)
-
-        all_labels = torch.cat(all_labels, dim=0)
-        mosaic_image = mosaic_image.resize((img_sz, img_sz))
-        return mosaic_image, all_labels
-
-
-class MixUp:
-    """Applies the MixUp augmentation to a pair of images and their corresponding boxes."""
-
-    def __init__(self, prob=0.5, alpha=1.0):
-        self.alpha = alpha
-        self.prob = prob
-        self.parent = None
-
-    def set_parent(self, parent):
-        """Set the parent dataset object for accessing dataset methods."""
-        self.parent = parent
-
-    def __call__(self, image, boxes):
-        if torch.rand(1) >= self.prob:
-            return image, boxes
-
-        assert self.parent is not None, "Parent is not set. MixUp cannot retrieve additional data."
-
-        # Retrieve another image and its boxes randomly from the dataset
-        image2, boxes2 = self.parent.get_more_data()[0]
-
-        # Calculate the mixup lambda parameter
-        lam = np.random.beta(self.alpha, self.alpha) if self.alpha > 0 else 0.5
-
-        # Mix images
-        image1, image2 = TF.to_tensor(image), TF.to_tensor(image2)
-        mixed_image = lam * image1 + (1 - lam) * image2
-
-        # Mix bounding boxes
-        mixed_boxes = torch.cat([lam * boxes, (1 - lam) * boxes2])
-
-        return TF.to_pil_image(mixed_image), mixed_boxes

utils/dataloader.py

@@ -1,186 +0,0 @@
-from os import listdir, path
-from typing import List, Tuple, Union
-
-import diskcache as dc
-import hydra
-import numpy as np
-import torch
-from loguru import logger
-from PIL import Image
-from torch.utils.data import DataLoader, Dataset
-from torchvision.transforms import functional as TF
-from tqdm.rich import tqdm
-
-from utils.data_augment import Compose, HorizontalFlip, MixUp, Mosaic, VerticalFlip
-from utils.drawer import draw_bboxes
-
-
-class YoloDataset(Dataset):
-    def __init__(self, config: dict, phase: str = "train", image_size: int = 640):
-        dataset_cfg = config.data
-        augment_cfg = config.augmentation
-        phase_name = dataset_cfg.get(phase, phase)
-        self.image_size = image_size
-
-        transforms = [eval(aug)(prob) for aug, prob in augment_cfg.items()]
-        self.transform = Compose(transforms, self.image_size)
-        self.transform.get_more_data = self.get_more_data
-        self.data = self.load_data(dataset_cfg.path, phase_name)
-
-    def load_data(self, dataset_path, phase_name):
-        """
-        Loads data from a cache or generates a new cache for a specific dataset phase.
-
-        Parameters:
-            dataset_path (str): The root path to the dataset directory.
-            phase_name (str): The specific phase of the dataset (e.g., 'train', 'test') to load or generate data for.
-
-        Returns:
-            dict: The loaded data from the cache for the specified phase.
-        """
-        cache_path = path.join(dataset_path, ".cache")
-        cache = dc.Cache(cache_path)
-        data = cache.get(phase_name)
-
-        if data is None:
-            logger.info("Generating {} cache", phase_name)
-            images_path = path.join(dataset_path, phase_name, "images")
-            labels_path = path.join(dataset_path, phase_name, "labels")
-            data = self.filter_data(images_path, labels_path)
-            cache[phase_name] = data
-
-        cache.close()
-        logger.info("📦 Loaded {} cache", phase_name)
-        data = cache[phase_name]
-        return data
-
-    def filter_data(self, images_path: str, labels_path: str) -> list:
-        """
-        Filters and collects dataset information by pairing images with their corresponding labels.
-
-        Parameters:
-            images_path (str): Path to the directory containing image files.
-            labels_path (str): Path to the directory containing label files.
-
-        Returns:
-            list: A list of tuples, each containing the path to an image file and its associated labels as a tensor.
-        """
-        data = []
-        valid_inputs = 0
-        images_list = sorted(listdir(images_path))
-        for image_name in tqdm(images_list, desc="Filtering data"):
-            if not image_name.lower().endswith((".jpg", ".jpeg", ".png")):
-                continue
-
-            img_path = path.join(images_path, image_name)
-            base_name, _ = path.splitext(image_name)
-            label_path = path.join(labels_path, f"{base_name}.txt")
-
-            if path.isfile(label_path):
-                labels = self.load_valid_labels(label_path)
-                if labels is not None:
-                    data.append((img_path, labels))
-                    valid_inputs += 1
-
-        logger.info("Recorded {}/{} valid inputs", valid_inputs, len(images_list))
-        return data
-
-    def load_valid_labels(self, label_path: str) -> Union[torch.Tensor, None]:
-        """
-        Loads and validates bounding box data is [0, 1] from a label file.
-
-        Parameters:
-            label_path (str): The filepath to the label file containing bounding box data.
-
-        Returns:
-            torch.Tensor or None: A tensor of all valid bounding boxes if any are found; otherwise, None.
-        """
-        bboxes = []
-        with open(label_path, "r") as file:
-            for line in file:
-                parts = list(map(float, line.strip().split()))
-                cls = parts[0]
-                points = np.array(parts[1:]).reshape(-1, 2)
-                valid_points = points[(points >= 0) & (points <= 1)].reshape(-1, 2)
-                if valid_points.size > 1:
-                    bbox = torch.tensor([cls, *valid_points.min(axis=0), *valid_points.max(axis=0)])
-                    bboxes.append(bbox)
-
-        if bboxes:
-            return torch.stack(bboxes)
-        else:
-            logger.warning("No valid BBox in {}", label_path)
-            return None
-
-    def get_data(self, idx):
-        img_path, bboxes = self.data[idx]
-        img = Image.open(img_path).convert("RGB")
-        return img, bboxes
-
-    def get_more_data(self, num: int = 1):
-        indices = torch.randint(0, len(self), (num,))
-        return [self.get_data(idx) for idx in indices]
-
-    def __getitem__(self, idx) -> Union[Image.Image, torch.Tensor]:
-        img, bboxes = self.get_data(idx)
-        if self.transform:
-            img, bboxes = self.transform(img, bboxes)
-        img = TF.to_tensor(img)
-        return img, bboxes
-
-    def __len__(self) -> int:
-        return len(self.data)
-
-
-class YoloDataLoader(DataLoader):
-    def __init__(self, config: dict):
-        """Initializes the YoloDataLoader with hydra-config files."""
-        hyper = config.hyper.data
-        dataset = YoloDataset(config)
-
-        super().__init__(
-            dataset,
-            batch_size=hyper.batch_size,
-            shuffle=hyper.shuffle,
-            num_workers=hyper.num_workers,
-            pin_memory=hyper.pin_memory,
-            collate_fn=self.collate_fn,
-        )
-
-    def collate_fn(self, batch: List[Tuple[torch.Tensor, torch.Tensor]]) -> Tuple[torch.Tensor, List[torch.Tensor]]:
-        """
-        A collate function to handle batching of images and their corresponding targets.
-
-        Args:
-            batch (list of tuples): Each tuple contains:
-                - image (torch.Tensor): The image tensor.
-                - labels (torch.Tensor): The tensor of labels for the image.
-
-        Returns:
-            Tuple[torch.Tensor, List[torch.Tensor]]: A tuple containing:
-                - A tensor of batched images.
-                - A list of tensors, each corresponding to bboxes for each image in the batch.
-        """
-        images = torch.stack([item[0] for item in batch])
-        targets = [item[1] for item in batch]
-        return images, targets
-
-
-def get_dataloader(config):
-    return YoloDataLoader(config)
-
-
-@hydra.main(config_path="../config", config_name="config", version_base=None)
-def main(cfg):
-    dataloader = get_dataloader(cfg)
-    draw_bboxes(next(iter(dataloader)))
-
-
-if __name__ == "__main__":
-    import sys
-
-    sys.path.append("./")
-    from tools.log_helper import custom_logger
-
-    custom_logger()
-    main()

utils/drawer.py

@@ -1,41 +0,0 @@
-from typing import List, Union
-
-import torch
-from loguru import logger
-from PIL import Image, ImageDraw, ImageFont
-from torchvision.transforms.functional import to_pil_image
-
-
-def draw_bboxes(img: Union[Image.Image, torch.Tensor], bboxes: List[List[Union[int, float]]]):
-    """
-    Draw bounding boxes on an image.
-
-    Args:
-    - img (PIL Image or torch.Tensor): Image on which to draw the bounding boxes.
-    - bboxes (List of Lists/Tensors): Bounding boxes with [class_id, x_min, y_min, x_max, y_max],
-      where coordinates are normalized [0, 1].
-    """
-    # Convert tensor image to PIL Image if necessary
-    if isinstance(img, torch.Tensor):
-        if img.dim() > 3:
-            logger.info("Multi-frame tensor detected, using the first image.")
-            img = img[0]
-            bboxes = bboxes[0]
-        img = to_pil_image(img)
-
-    draw = ImageDraw.Draw(img)
-    width, height = img.size
-    font = ImageFont.load_default(30)
-
-    for bbox in bboxes:
-        class_id, x_min, y_min, x_max, y_max = bbox
-        x_min = x_min * width
-        x_max = x_max * width
-        y_min = y_min * height
-        y_max = y_max * height
-        shape = [(x_min, y_min), (x_max, y_max)]
-        draw.rectangle(shape, outline="red", width=3)
-        draw.text((x_min, y_min), str(int(class_id)), font=font, fill="blue")
-
-    img.save("visualize.jpg")  # Save the image with annotations
-    logger.info("Saved visualize image at visualize.png")

utils/get_dataset.py

@@ -1,84 +0,0 @@
-import os
-import zipfile
-
-import hydra
-import requests
-from loguru import logger
-from tqdm.rich import tqdm
-
-
-def download_file(url, dest_path):
-    """
-    Downloads a file from a specified URL to a destination path with progress logging.
-    """
-    logger.info(f"Downloading {os.path.basename(dest_path)}...")
-    with requests.get(url, stream=True) as r:
-        r.raise_for_status()
-        total_length = int(r.headers.get("content-length", 0))
-        with open(dest_path, "wb") as f, tqdm(
-            total=total_length, unit="iB", unit_scale=True, desc=os.path.basename(dest_path), leave=True
-        ) as bar:
-            for chunk in r.iter_content(chunk_size=1024 * 1024):
-                f.write(chunk)
-                bar.update(len(chunk))
-    logger.info("Download complete!")
-
-
-def unzip_file(zip_path, extract_to):
-    """
-    Unzips a ZIP file to a specified directory.
-    """
-    logger.info(f"Unzipping {os.path.basename(zip_path)}...")
-    with zipfile.ZipFile(zip_path, "r") as zip_ref:
-        zip_ref.extractall(extract_to)
-    os.remove(zip_path)
-    logger.info(f"Removed {zip_path}")
-
-
-def check_files(directory, expected_count):
-    """
-    Checks if the specified directory has the expected number of files.
-    """
-    num_files = len([name for name in os.listdir(directory) if os.path.isfile(os.path.join(directory, name))])
-    return num_files == expected_count
-
-
-@hydra.main(config_path="../config/data", config_name="download", version_base=None)
-def prepare_dataset(download_cfg):
-    data_dir = download_cfg.path
-    base_url = download_cfg.images.base_url
-    datasets = download_cfg.images.datasets
-
-    for dataset_type in datasets:
-        file_name, expected_files = datasets[dataset_type].values()
-        url = f"{base_url}{file_name}"
-        local_zip_path = os.path.join(data_dir, file_name)
-        extract_to = os.path.join(data_dir, dataset_type, "images")
-
-        # Ensure the extraction directory exists
-        os.makedirs(extract_to, exist_ok=True)
-
-        # Check if the correct number of files exists
-        if check_files(extract_to, expected_files):
-            logger.info(f"✅ Dataset {dataset_type: >4} already verified.")
-            continue
-
-        if os.path.exists(local_zip_path):
-            logger.info(f"Dataset {dataset_type} already downloaded.")
-        else:
-            download_file(url, local_zip_path)
-
-        unzip_file(local_zip_path, extract_to)
-
-        print(os.path.exists(local_zip_path), check_files(extract_to, expected_files))
-
-        # Additional verification post extraction
-        if not check_files(extract_to, expected_files):
-            logger.error(f"Error in verifying the {dataset_type} dataset after extraction.")
-
-
-if __name__ == "__main__":
-    from tools.log_helper import custom_logger
-
-    custom_logger()
-    prepare_dataset()

utils/loss.py

@@ -1,2 +0,0 @@
-def get_loss_function(*args, **kwargs):
-    raise NotImplementedError