utils/data_processing.py

"""Provides the standard data processing functions performed on CNMFe and annotation data"""
import numpy as np
from scipy.signal import correlate
from scipy.stats import zscore

def smooth(data: np.ndarray, window_size=5):
    """
    Returns a smoothed version of response data using a moving average filter.

    Parameters:
    ----------
    data : np.ndarray
        A numpy 1-D array containing data to be smoothed.
    window_size : int 
        Number of data points for calculating the smoothed value. If an even number is
        passed in, window_size is autmoatically reduced by 1.

    Returns:
    --------
    smooth_data : np.ndarray
        Smoothed data, returned as a 1-D array of the same size as ``data``.
    
    Notes:
    ------
    Implements MATLAB's smooth function.
    """
    if window_size == 0:
        raise ValueError('window_size can not be 0.')
    if window_size == 1:
        return data
    if window_size > data.size:
        window_size = data.size
    if window_size%2 == 0:
        window_size = window_size - 1
    outside_valid_window_size = int((window_size-1)/2)
    start = np.array([np.sum(data[0:(2*k+1)]/(2*k+1)) for k in range(outside_valid_window_size)])
    end   = np.array([np.sum(data[-(2*k+1):]/(2*k+1)) for k in range(outside_valid_window_size)])[::-1]
    smoothed_data = np.convolve(data,np.ones(window_size,dtype=int),'valid')/window_size
    return np.hstack((start,smoothed_data,end))

def corr(x: np.ndarray, y: np.ndarray):
    """
    Returns a matrix of the pairwise correlation coefficient between each pair of columns
    in the input matrices x and y.

    Parameters:
    -----------
    x : np.ndarray
        Input matrix, specified as an n x k_1 matrix. Its rows correspond to
        observations, and the columns correspond to variables.
    y : np.ndarray
        Input matrix, specified as an n x k_2 matrix. Its rows correspond to
        observations, and the columns correspond to variables.

    Returns:
    --------
    rho - Pairwise linear correlation coefficient, returned as a matrix.
    
    Notes:
    ------
    Implements MATLAB's corr function.
    """
    return np.corrcoef(x,y)[0][1]

def autocorr(x:np.ndarray,
             max_lags=10):
    """
    Returns the correlations and associated lags of the univariate time series x.

    Parameters:
    -----------
    x : np.ndarray
        Observed univariate time series.
    max_lags : int 
        Number of lags, specified as a positive integer.

    Returns:
    acf : np.ndarray
        Correlations, returned as a numeric vector of length ``max_lags`` + 1.
    lags : np.ndarray
        Autocorrelation lags.

    Notes:
    ------
    Modified version of matplotlib's acorr function.
    """
    Nx = len(x)

    correls = correlate(x, x, mode="full")
    correls = correls / np.dot(x, x)

    if max_lags is None:
        max_lags = Nx - 1

    if max_lags >= Nx or max_lags < 1:
        raise ValueError('maxlags must be None or strictly '
                            'positive < %d' % Nx)

    lags = np.arange(-max_lags, max_lags + 1)
    acf = correls[Nx - 1 - max_lags:Nx + max_lags]

    return acf, lags

def convert_to_rast(behavior_ts, time_max):
    """
    Converts a list of behavior time stamps to a one-hot vector where 0 indicates no
    presence of the given behavior, and 1 indicates presence of it.

    Args:
        behavior_ts   - a list of time stamps (start and end) for a particular behavior\n
        time_max      - the length in frames of the vector
    
    Returns:
        behavior_rast - a one-hot vector
    """
    behavior_rast = np.zeros(time_max)
    for time_stamps in behavior_ts:
        start = int(round(time_stamps[0]))
        end   = int(round(time_stamps[1] + 1))
        if start > time_max:
            break
        if end > time_max:
            end = time_max
        np.put(behavior_rast,range(start,end),np.ones(end-start))
    return behavior_rast

def convert_to_raster(bouts: list,
                      neural_activity_sr: float,
                      observation_sr: float,
                      max_frame: int):
    """
    Converts bouts into a behavior raster, a one hot encoding of a behavior describing
    when it is active.

    It is often the case that the start and stop timestamps found in ``bouts`` are
    collected at a different sample rate than ``neural_activity``, which are often what
    behavior rasters align to. In order to align the two, a ratio between the sample
    rates of ``neural_activity`` and the bouts of behavior, which are observations,
    is calculated and then multiplied to the timestamps.

    Parameters:
    -----------
    bouts : np.ndarray
        An array where each element is a pair of integers where the first integer denotes
        the beginning of a bout of behavior, and the second integer denotes the end of
        the bout.
    neural_activity_sr : float
        Sample rate of ``neural_activity``.
    observation_sr : float
        Sample rate for the ``bouts`` used.
    max_frame : int
        The length of the behavior raster, often set to the number of frames of
        ``neural_activity``.
    
    Returns:
    --------
    behavior_raster : np.ndarray
        A raster (a one hot encoding) of a behavior, describing when it is active.
    """
    sr_ratio             = neural_activity_sr/observation_sr
    behavior_ts_adjusted = bouts*sr_ratio
    behavior_raster      = np.zeros(max_frame)
    for time_stamps in behavior_ts_adjusted:
        start = int(round(time_stamps[0]))
        end   = int(round(time_stamps[1] + 1))
        if start > max_frame:
            break
        if end > max_frame:
            end = max_frame
        np.put(behavior_raster,range(start,end),np.ones(end-start))
    return behavior_raster

def convert_to_bouts(behavior_raster: np.ndarray):
    """
    Converts a behavior raster into behavior bouts, an array where each element is a
    pair of timestamps (int) where the first timestamp denotes the beginning of a bout of
    behavior, and the second timestamp denotes the end of the bout.

    Parameters:
    -----------
    behavior_raster : np.ndarray
        A raster (a one hot encoding) of a behavior, describing when it is active.

    Returns:
    --------
    bouts : np.ndarray
        An array where each element is a pair of timestamps (int) where the first
        timestamp denotes the beginning of a bout of behavior, and the second timestamp
        denotes the end of the bout.
    """
    dt = behavior_raster[1:] - behavior_raster[:-1]
    start = np.where(dt==1)[0] + 1
    stop  = np.where(dt==-1)[0]
    if behavior_raster[0]:
        start = np.concatenate((np.array([0]),start))
    if behavior_raster[-1]:
        stop = np.concatenate((stop,[behavior_raster.size]))
    bouts = np.hstack((np.reshape(start,(len(start),1)),
                       np.reshape(stop,(len(stop),1))))
    return bouts

def merge_rasters_down(behavior_raster_array: np.ndarray)-> np.ndarray:
    """
    For a behavior raster, merges down all rasters to one array in such a way that no
    two behaviors are occuring at the same time.

    It determines which behavior should remain 'on top' by determening which behavior
    has the least amount of active frames.

    This method should only be used on behavior rasters where all behaviors come from a
    single channel.

    Parameters:
    -----------
    behavior_raster_array : np.ndarray
        An array where each row is a behavior raster, a one hot encoding of behaviors,
        describing when that behavior is active. Each row of this array must use a
        different value to indicate that a behavior is active (for example, if one
        row uses 1s, another row must not use 1 as well).
    
    Returns:
    --------
    single_track : np.ndarray
        An array which is the length of a behavior raster in ``behavior_raster_array``,
        where each entry is either 0 indicating that no behavior is active, or a value
        indicating that a specific behavior is active.
    """
    # single track
    single_track = np.zeros((1,behavior_raster_array.shape[1]))

    # determine order to insert row values
    num_active_frames = [np.sum(np.where(row > 0, 1, 0)) for row in behavior_raster_array]

    for i in range(behavior_raster_array.shape[0]):
        max_i = np.argmax(num_active_frames)
        num_active_frames[max_i] = -1

        unique_values = np.unique(behavior_raster_array[max_i])
        if len(unique_values) > 1: value = unique_values[1]
        else: value = 0
        active_inds = np.where(behavior_raster_array[max_i] == value)[0]

        single_track[:,active_inds] = value
    return single_track

def separate_tracks(single_track: np.ndarray,
                    behavior_values: list):
    """
    For a single track, separates each unique value (except for 0) into its own raster
    within a 2-D array.

    Parameters:
    -----------
    single_track : np.ndarray
        An array which is the length of a behavior raster in ``behavior_raster_array``,
        where each entry is either 0 indicating that no behavior is active, or a value
        indicating that a specific behavior is active.
    behavior_values : list
        A list of values corresponding to the specific behaviors within ``single_track``.
    
    Returns:
    --------
    behavior_raster_array : np.ndarray
        An array where each row is a behavior raster, a one hot encoding of behaviors,
        describing when that behavior is active.
    """
    if len(behavior_values) < np.unique(single_track).size - 1:
        raise KeyError("There are not sufficient values within ``behavior_values`` to "
                       "accomodate those present in ``single_track``.")
    tracks = []
    for value in behavior_values:
        tracks.append(np.where(single_track == value, value, 0))
    return np.vstack(tracks)

def config_neural_activity(config: dict, neural_activity: np.ndarray):
    """
    Configures `neural_activity` according to parameters set in config.

    Parameters:
    -----------
    config : dict
        A dictionary which specifies the following parameters: 'smooth_window',
        'baseline_frame', and 'zscore_method'. 'zscore_method' is one of "All Data",
        "Baseline", or "No Z-Score".
    neural_activity : np.ndarray
        Neural activity being used.
    
    Returns:
    --------
    mod_neural_activity : np.ndarray
        Modified `neural_activity`, accodring to `config`.
    """
    smooth_window  = config['smooth_window']
    zscore_method  = config['zscore_method']
    baseline_frame = config['baseline_frame']

    # smooth
    if len(neural_activity.shape) > 1:
        neural_data_smooth = np.zeros(neural_activity.shape)
        for i in range(neural_activity.shape[0]):
            neural_data_smooth[i] = smooth(neural_activity[i], int(smooth_window))
        mod_neural_activity = neural_data_smooth
    else:
        mod_neural_activity = smooth(neural_activity, int(smooth_window))

    # z-score
    if zscore_method == 'Baseline' and (not baseline_frame is None or baseline_frame == 0):
        if len(neural_activity.shape)> 1:
            mean = mod_neural_activity[:,:baseline_frame].mean(axis=1,keepdims=True)
            std  = mod_neural_activity[:,:baseline_frame].std(axis=1,keepdims=True)
        else:
            mean = mod_neural_activity[:baseline_frame].mean()
            std  = mod_neural_activity[:baseline_frame].std()
        mod_neural_activity = (mod_neural_activity - mean) / std
    elif zscore_method == 'No Z-Score':
        mod_neural_activity = mod_neural_activity
    else:
        if len(neural_activity.shape) > 1:
            mod_neural_activity = zscore(mod_neural_activity,axis=1)
        else:
            mod_neural_activity = zscore(mod_neural_activity)
    return mod_neural_activity

def compress_annotations(annot: dict, downsample_rate: int, max_frame: int)-> dict:
    """
    Takes in an annotation dictionary and creates a single raster per channel, where the
    raster contains the behaviors from their respective channel.

    annot : dict
        Dictionary of beginning and end frames for behaviors.
    downsample_rate : int
        The rate at which samples should be taken. Divides bout timing (in frames) by
        value.
    max_frame : int
        The last frame for annotations from `annot`.
    """
    annot_single_track = {}
    channel_behavior_map = {}
    for channel in annot:
        channel_rasters = []
        behavior_map = {}
        behavior_map.update({0: 'None'})
        for i, behavior in enumerate(annot[channel]):
            bouts = annot[channel][behavior]
            raster = convert_to_raster(bouts, 1, downsample_rate, max_frame)
            channel_rasters.append(raster*(i+1))
            behavior_map.update({(i+1) : behavior})
        channel_raster = merge_rasters_down(np.array(channel_rasters))[0]
        annot_single_track.update({channel : channel_raster})
        channel_behavior_map.update({channel : behavior_map})
    return annot_single_track, channel_behavior_map

def compress_compressed_annotations(annot_single_track: dict,
                                    channel_behavior_map: dict,
                                    max_frame: int):
    """
    Further compresses the results from `compress_annotations` to get a single array
    where each entry is a list of the behaviors present at that frame across all channels.
    """
    labels = []
    for frame in range(max_frame):
        labels_at_frame = []
        for channel in annot_single_track:
            channel_raster = annot_single_track[channel]
            behavior_map = channel_behavior_map[channel]
            behavior_value = int(channel_raster[frame])
            behavior_label = behavior_map.get(behavior_value)
            labels_at_frame.append(behavior_label)
        labels.append('||'.join(labels_at_frame))
    return labels

def generate_label_array(annot: dict,
                         downsample_rate: int,
                         max_frame: int)-> list[str]:
    """
    Generates an array of lists of labels, where each entry is a video frame, and the
    labels come from each channel in `annot`.
    """
    annot_single_track,\
    channel_behavior_map = compress_annotations(annot, downsample_rate, max_frame)
    labels = compress_compressed_annotations(annot_single_track,
                                             channel_behavior_map,
                                             max_frame)
    return labels