face_deid_ct.py

import os
import pydicom
import numpy as np
import cv2
from matplotlib import pyplot as plt
import random
import time
import tqdm
from IPython.core.display import display, HTML

# Determine if we are in a Jupyter notebook
try:
    shell = get_ipython().__class__.__name__
    if shell == 'ZMQInteractiveShell':
        # We are in Jupyter, use tqdm.notebook
        from tqdm.notebook import tqdm
    else:
        raise Exception()
except:
    # We are in a terminal, use standard tqdm
    from tqdm import tqdm

    
FACE_MAX_VALUE = 50
FACE_MIN_VALUE = -125

AIR_THRESHOLD  = -800
KERNEL_SIZE    = 35



def is_dicom(file_path):
    try:
        pydicom.dcmread(file_path)
        return True
    except Exception:
        return False

def get_first_directory(path):
    # Normalize the path to always use Unix-style path separators
    normalized_path = path.replace("\\", "/")
    split_path = normalized_path.split("/")[-1]
    
    return split_path  # Return None if no directories are found

def list_dicom_directories(root_dir):
    dicom_dirs = set()
    
    for root, dirs, files in os.walk(root_dir):
        for file in files:
            file_path = os.path.join(root, file)
            if is_dicom(file_path):
                dicom_dirs.add(root)
                break
                
    return list(dicom_dirs)

def load_scan(path):
    slices = [pydicom.read_file(path + '/' + s) for s in os.listdir(path)]
    slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))
    try:
        slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])
    except:
        try:
            slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)
        except:
            slice_thickness = 1.0

    for s in slices:
        s.SliceThickness = slice_thickness
        
    return slices

def get_pixels_hu(slices):
    image = np.stack([s.pixel_array for s in slices])
    # Convert to int16 (from sometimes int16), 
    # should be possible as values should always be low enough (<32k)
    image = image.astype(np.int16)

    # Set outside-of-scan pixels to 0
    # The intercept is usually -1024, so air is approximately 0
    image[image == -2000] = 0
    
    # Convert to Hounsfield units (HU)
    for slice_number in range(len(slices)):
        
        intercept = slices[slice_number].RescaleIntercept
        slope = slices[slice_number].RescaleSlope
        
        if slope != 1:
            image[slice_number] = slope * image[slice_number].astype(np.float64)
            image[slice_number] = image[slice_number].astype(np.int16)
            
        image[slice_number] += np.int16(intercept)
    
    return np.array(image, dtype=np.int16)

def binarize_volume(volume, air_hu=AIR_THRESHOLD):
    binary_volume = np.zeros_like(volume, dtype=np.uint8)
    binary_volume[volume <= air_hu] = 1
    return binary_volume

def largest_connected_component(binary_image):
    # Find all connected components and stats
    num_labels, labels, stats, centroids = cv2.connectedComponentsWithStats(binary_image, connectivity=8)

    # Get the index of the largest component, ignoring the background
    # The background is considered as a component by connectedComponentsWithStats and it is usually the first component
    largest_component_index = np.argmax(stats[1:, cv2.CC_STAT_AREA]) + 1

    # Create an image to keep largest component only
    largest_component_image = np.zeros(labels.shape, dtype=np.uint8)
    largest_component_image[labels == largest_component_index] = 1

    return largest_component_image

def get_largest_component_volume(volume):
    # Initialize an empty array to hold the processed volume
    processed_volume = np.empty_like(volume, dtype=np.uint8)
    
    # Iterate over each slice in the volume
    for i in range(volume.shape[0]):
        # Process the slice and store it in the processed volume
        processed_volume[i] = largest_connected_component(volume[i])
        
    return processed_volume



def dilate_volume(volume, kernel_size=KERNEL_SIZE):
    # Create the structuring element (kernel) for dilation
    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size))
    
    # Initialize an empty array to hold the dilated volume
    dilated_volume = np.empty_like(volume)
    
    # Iterate over each slice in the volume
    for i in range(volume.shape[0]):
        # Dilate the slice and store it in the dilated volume
        dilated_volume[i] = cv2.dilate(volume[i].astype(np.uint8), kernel)
        
    return dilated_volume


def apply_mask_and_get_values(image_volume, mask_volume):
    # Apply the mask by multiplying the image volume with the mask volume
    masked_volume = image_volume * mask_volume
    
    # Get all unique values in the masked volume, excluding zero
    unique_values = np.unique(masked_volume)
    unique_values = unique_values[unique_values > FACE_MIN_VALUE]
    unique_values = unique_values[unique_values < FACE_MAX_VALUE]

    # Convert numpy array to a list
    unique_values_list = unique_values.tolist()
    
    return unique_values_list


def apply_random_values_optimized(pixels_hu, dilated_volume, unique_values_list):
    # Initialize new volume as a copy of the original volume
    new_volume = np.copy(pixels_hu)

    # Generate random indices
    random_indices = np.random.choice(len(unique_values_list), size=np.sum(dilated_volume))

    # Select random values from the unique_values_list
    random_values = np.array(unique_values_list)[random_indices]

    # Apply the random values to the locations where dilated_volume equals 1
    new_volume[dilated_volume == 1] = random_values

    return new_volume

def save_new_dicom_files(new_volume, original_dir, out_path, app="_d"):
    # Create a new directory path by appending "_d" to the original directory
    if out_path is None:
        new_dir = original_dir + app
    else:
        new_dir = out_path

    # Create the new directory if it doesn't exist
    if not os.path.exists(new_dir):
        os.makedirs(new_dir)

    # List all DICOM files in the original directory
    dicom_files = [os.path.join(original_dir, f) for f in os.listdir(original_dir) if f.endswith('.dcm')]

    # Sort the dicom_files list by SliceLocation
    dicom_files.sort(key=lambda x: pydicom.dcmread(x).SliceLocation)

    # Loop over each slice of the new volume
    for i in range(new_volume.shape[0]):
        # Get the corresponding original DICOM file
        dicom_file = dicom_files[i]

        # Read the file
        ds = pydicom.dcmread(dicom_file)
        ds.decompress()

        # Revert the slope and intercept operation on the slice
        new_slice = (new_volume[i] - ds.RescaleIntercept) / ds.RescaleSlope

        # Update the pixel data with the data from the new slice
        ds.PixelData = new_slice.astype(np.int16).tobytes()

        # Generate new file name 
        new_file_name = os.path.join(new_dir, f"new_image_{i}.dcm")

        # Save the new DICOM file
        ds.save_as(new_file_name)



def drown_volume(in_path, out_path='deid_ct', replacer='face'):
    """
    Processes DICOM files from the provided directory by binarizing, getting the largest connected component, 
    dilating and applying mask. Then applies random values to the dilated volume based on a unique values list 
    obtained from the masked volume (or air value). The results are saved as new DICOM files in a specified directory.
    
    Parameters:
    in_path (str): The path to the directory containing the input DICOM files.
    out_path (str, optional): The path to the directory where the output DICOM files will be saved. 
                              If not provided, the output files will be saved in the input directory appended by "_d".
    replacer (str, optional): Indicates what kind of pixels are going to be replaced. Default is 'face'.
                              'face': replaces air and face with random values that are found in the skin and subcutaneous fat.
                              'air': replaces air and face with -1000 HU.
                              int: replaces air and face with int HU.
    
    Returns:
    None. The function saves new DICOM files and prints the total elapsed time of the operation.
    """
    start_time = time.time()

    dirs = list_dicom_directories(in_path)
    
    for _d in tqdm(dirs, desc="List of studies"):

        with tqdm(total=8, desc="Processing DICOM Files", leave=False) as pbar:
            # Load the DICOM files
            slices = load_scan(_d)
            pbar.update()

            # Get the pixel values and convert them to Hounsfield Units (HU)
            pixels_hu = get_pixels_hu(slices)
            pbar.update()

            # Apply the binarization function on the HU volume
            binarized_volume = binarize_volume(pixels_hu)
            pbar.update()

            # Get the largest connected component from the binarized volume
            processed_volume = get_largest_component_volume(binarized_volume)
            pbar.update()

            # Dilate the processed volume
            dilated_volume = dilate_volume(processed_volume)
            pbar.update()
            if replacer == 'face':
                # Apply the mask to the original volume and get unique values list
                unique_values_list = apply_mask_and_get_values(pixels_hu, dilated_volume - processed_volume)
            elif replacer == 'air':
                unique_values_list = [0]
            else:
                try:
                    replacer = int(replacer)
                    unique_values_list = [replacer]
                except:
                    print('replacer must be either air, face, or an integer number in Hounsfield units, but ' + str(replacer) + ' was provided.')
                    print('replacing with face')
                    unique_values_list = apply_mask_and_get_values(pixels_hu, dilated_volume - processed_volume)

            pbar.update()

            # Apply random values to the dilated volume based on the unique values list
            new_volume = apply_random_values_optimized(pixels_hu, dilated_volume, unique_values_list)
            pbar.update()

            # Save the new DICOM files
            out_path_n = out_path + "/" + get_first_directory(_d)
            save_new_dicom_files(new_volume, _d, out_path_n)
            pbar.update()

        elapsed_time = time.time() - start_time
        print(f"Total elapsed time: {elapsed_time} seconds")