"""Provides the standard data processing functions performed on CNMFe and annotation data""" import numpy as np from scipy.signal import correlate from scipy.stats import zscore def smooth(data: np.ndarray, window_size=5): """ Returns a smoothed version of response data using a moving average filter. Parameters: ---------- data : np.ndarray A numpy 1-D array containing data to be smoothed. window_size : int Number of data points for calculating the smoothed value. If an even number is passed in, window_size is autmoatically reduced by 1. Returns: -------- smooth_data : np.ndarray Smoothed data, returned as a 1-D array of the same size as ``data``. Notes: ------ Implements MATLAB's smooth function. """ if window_size == 0: raise ValueError('window_size can not be 0.') if window_size == 1: return data if window_size > data.size: window_size = data.size if window_size%2 == 0: window_size = window_size - 1 outside_valid_window_size = int((window_size-1)/2) start = np.array([np.sum(data[0:(2*k+1)]/(2*k+1)) for k in range(outside_valid_window_size)]) end = np.array([np.sum(data[-(2*k+1):]/(2*k+1)) for k in range(outside_valid_window_size)])[::-1] smoothed_data = np.convolve(data,np.ones(window_size,dtype=int),'valid')/window_size return np.hstack((start,smoothed_data,end)) def corr(x: np.ndarray, y: np.ndarray): """ Returns a matrix of the pairwise correlation coefficient between each pair of columns in the input matrices x and y. Parameters: ----------- x : np.ndarray Input matrix, specified as an n x k_1 matrix. Its rows correspond to observations, and the columns correspond to variables. y : np.ndarray Input matrix, specified as an n x k_2 matrix. Its rows correspond to observations, and the columns correspond to variables. Returns: -------- rho - Pairwise linear correlation coefficient, returned as a matrix. Notes: ------ Implements MATLAB's corr function. """ return np.corrcoef(x,y)[0][1] def autocorr(x:np.ndarray, max_lags=10): """ Returns the correlations and associated lags of the univariate time series x. Parameters: ----------- x : np.ndarray Observed univariate time series. max_lags : int Number of lags, specified as a positive integer. Returns: acf : np.ndarray Correlations, returned as a numeric vector of length ``max_lags`` + 1. lags : np.ndarray Autocorrelation lags. Notes: ------ Modified version of matplotlib's acorr function. """ Nx = len(x) correls = correlate(x, x, mode="full") correls = correls / np.dot(x, x) if max_lags is None: max_lags = Nx - 1 if max_lags >= Nx or max_lags < 1: raise ValueError('maxlags must be None or strictly ' 'positive < %d' % Nx) lags = np.arange(-max_lags, max_lags + 1) acf = correls[Nx - 1 - max_lags:Nx + max_lags] return acf, lags def convert_to_rast(behavior_ts, time_max): """ Converts a list of behavior time stamps to a one-hot vector where 0 indicates no presence of the given behavior, and 1 indicates presence of it. Args: behavior_ts - a list of time stamps (start and end) for a particular behavior\n time_max - the length in frames of the vector Returns: behavior_rast - a one-hot vector """ behavior_rast = np.zeros(time_max) for time_stamps in behavior_ts: start = int(round(time_stamps[0])) end = int(round(time_stamps[1] + 1)) if start > time_max: break if end > time_max: end = time_max np.put(behavior_rast,range(start,end),np.ones(end-start)) return behavior_rast def convert_to_raster(bouts: list, neural_activity_sr: float, observation_sr: float, max_frame: int): """ Converts bouts into a behavior raster, a one hot encoding of a behavior describing when it is active. It is often the case that the start and stop timestamps found in ``bouts`` are collected at a different sample rate than ``neural_activity``, which are often what behavior rasters align to. In order to align the two, a ratio between the sample rates of ``neural_activity`` and the bouts of behavior, which are observations, is calculated and then multiplied to the timestamps. Parameters: ----------- bouts : np.ndarray An array where each element is a pair of integers where the first integer denotes the beginning of a bout of behavior, and the second integer denotes the end of the bout. neural_activity_sr : float Sample rate of ``neural_activity``. observation_sr : float Sample rate for the ``bouts`` used. max_frame : int The length of the behavior raster, often set to the number of frames of ``neural_activity``. Returns: -------- behavior_raster : np.ndarray A raster (a one hot encoding) of a behavior, describing when it is active. """ sr_ratio = neural_activity_sr/observation_sr behavior_ts_adjusted = bouts*sr_ratio behavior_raster = np.zeros(max_frame) for time_stamps in behavior_ts_adjusted: start = int(round(time_stamps[0])) end = int(round(time_stamps[1] + 1)) if start > max_frame: break if end > max_frame: end = max_frame np.put(behavior_raster,range(start,end),np.ones(end-start)) return behavior_raster def convert_to_bouts(behavior_raster: np.ndarray): """ Converts a behavior raster into behavior bouts, an array where each element is a pair of timestamps (int) where the first timestamp denotes the beginning of a bout of behavior, and the second timestamp denotes the end of the bout. Parameters: ----------- behavior_raster : np.ndarray A raster (a one hot encoding) of a behavior, describing when it is active. Returns: -------- bouts : np.ndarray An array where each element is a pair of timestamps (int) where the first timestamp denotes the beginning of a bout of behavior, and the second timestamp denotes the end of the bout. """ dt = behavior_raster[1:] - behavior_raster[:-1] start = np.where(dt==1)[0] + 1 stop = np.where(dt==-1)[0] if behavior_raster[0]: start = np.concatenate((np.array([0]),start)) if behavior_raster[-1]: stop = np.concatenate((stop,[behavior_raster.size])) bouts = np.hstack((np.reshape(start,(len(start),1)), np.reshape(stop,(len(stop),1)))) return bouts def merge_rasters_down(behavior_raster_array: np.ndarray)-> np.ndarray: """ For a behavior raster, merges down all rasters to one array in such a way that no two behaviors are occuring at the same time. It determines which behavior should remain 'on top' by determening which behavior has the least amount of active frames. This method should only be used on behavior rasters where all behaviors come from a single channel. Parameters: ----------- behavior_raster_array : np.ndarray An array where each row is a behavior raster, a one hot encoding of behaviors, describing when that behavior is active. Each row of this array must use a different value to indicate that a behavior is active (for example, if one row uses 1s, another row must not use 1 as well). Returns: -------- single_track : np.ndarray An array which is the length of a behavior raster in ``behavior_raster_array``, where each entry is either 0 indicating that no behavior is active, or a value indicating that a specific behavior is active. """ # single track single_track = np.zeros((1,behavior_raster_array.shape[1])) # determine order to insert row values num_active_frames = [np.sum(np.where(row > 0, 1, 0)) for row in behavior_raster_array] for i in range(behavior_raster_array.shape[0]): max_i = np.argmax(num_active_frames) num_active_frames[max_i] = -1 unique_values = np.unique(behavior_raster_array[max_i]) if len(unique_values) > 1: value = unique_values[1] else: value = 0 active_inds = np.where(behavior_raster_array[max_i] == value)[0] single_track[:,active_inds] = value return single_track def separate_tracks(single_track: np.ndarray, behavior_values: list): """ For a single track, separates each unique value (except for 0) into its own raster within a 2-D array. Parameters: ----------- single_track : np.ndarray An array which is the length of a behavior raster in ``behavior_raster_array``, where each entry is either 0 indicating that no behavior is active, or a value indicating that a specific behavior is active. behavior_values : list A list of values corresponding to the specific behaviors within ``single_track``. Returns: -------- behavior_raster_array : np.ndarray An array where each row is a behavior raster, a one hot encoding of behaviors, describing when that behavior is active. """ if len(behavior_values) < np.unique(single_track).size - 1: raise KeyError("There are not sufficient values within ``behavior_values`` to " "accomodate those present in ``single_track``.") tracks = [] for value in behavior_values: tracks.append(np.where(single_track == value, value, 0)) return np.vstack(tracks) def config_neural_activity(config: dict, neural_activity: np.ndarray): """ Configures `neural_activity` according to parameters set in config. Parameters: ----------- config : dict A dictionary which specifies the following parameters: 'smooth_window', 'baseline_frame', and 'zscore_method'. 'zscore_method' is one of "All Data", "Baseline", or "No Z-Score". neural_activity : np.ndarray Neural activity being used. Returns: -------- mod_neural_activity : np.ndarray Modified `neural_activity`, accodring to `config`. """ smooth_window = config['smooth_window'] zscore_method = config['zscore_method'] baseline_frame = config['baseline_frame'] # smooth if len(neural_activity.shape) > 1: neural_data_smooth = np.zeros(neural_activity.shape) for i in range(neural_activity.shape[0]): neural_data_smooth[i] = smooth(neural_activity[i], int(smooth_window)) mod_neural_activity = neural_data_smooth else: mod_neural_activity = smooth(neural_activity, int(smooth_window)) # z-score if zscore_method == 'Baseline' and (not baseline_frame is None or baseline_frame == 0): if len(neural_activity.shape)> 1: mean = mod_neural_activity[:,:baseline_frame].mean(axis=1,keepdims=True) std = mod_neural_activity[:,:baseline_frame].std(axis=1,keepdims=True) else: mean = mod_neural_activity[:baseline_frame].mean() std = mod_neural_activity[:baseline_frame].std() mod_neural_activity = (mod_neural_activity - mean) / std elif zscore_method == 'No Z-Score': mod_neural_activity = mod_neural_activity else: if len(neural_activity.shape) > 1: mod_neural_activity = zscore(mod_neural_activity,axis=1) else: mod_neural_activity = zscore(mod_neural_activity) return mod_neural_activity def compress_annotations(annot: dict, downsample_rate: int, max_frame: int)-> dict: """ Takes in an annotation dictionary and creates a single raster per channel, where the raster contains the behaviors from their respective channel. annot : dict Dictionary of beginning and end frames for behaviors. downsample_rate : int The rate at which samples should be taken. Divides bout timing (in frames) by value. max_frame : int The last frame for annotations from `annot`. """ annot_single_track = {} channel_behavior_map = {} for channel in annot: channel_rasters = [] behavior_map = {} behavior_map.update({0: 'None'}) for i, behavior in enumerate(annot[channel]): bouts = annot[channel][behavior] raster = convert_to_raster(bouts, 1, downsample_rate, max_frame) channel_rasters.append(raster*(i+1)) behavior_map.update({(i+1) : behavior}) channel_raster = merge_rasters_down(np.array(channel_rasters))[0] annot_single_track.update({channel : channel_raster}) channel_behavior_map.update({channel : behavior_map}) return annot_single_track, channel_behavior_map def compress_compressed_annotations(annot_single_track: dict, channel_behavior_map: dict, max_frame: int): """ Further compresses the results from `compress_annotations` to get a single array where each entry is a list of the behaviors present at that frame across all channels. """ labels = [] for frame in range(max_frame): labels_at_frame = [] for channel in annot_single_track: channel_raster = annot_single_track[channel] behavior_map = channel_behavior_map[channel] behavior_value = int(channel_raster[frame]) behavior_label = behavior_map.get(behavior_value) labels_at_frame.append(behavior_label) labels.append('||'.join(labels_at_frame)) return labels def generate_label_array(annot: dict, downsample_rate: int, max_frame: int)-> list[str]: """ Generates an array of lists of labels, where each entry is a video frame, and the labels come from each channel in `annot`. """ annot_single_track,\ channel_behavior_map = compress_annotations(annot, downsample_rate, max_frame) labels = compress_compressed_annotations(annot_single_track, channel_behavior_map, max_frame) return labels