Merge pull request #1800 from AllenInstitute/rc/2.5.0

rc/2.5.0

CHANGELOG.md

@@ -1,6 +1,10 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.5.0] = 2021-01-29
+-  Adds unfiltered running speed as new data stream
+-  run_demixing gracefully ignores any ROIs that are not in the input trace file
+
 ## [2.4.1] = 2021-01-04
 -  update deprecated call to scipy.spatial.transform.Rotation.as_dcm() to .as_matrix()
 

allensdk/__init__.py

@@ -37,7 +37,7 @@
 
 
 
-__version__ = '2.4.1'
+__version__ = '2.5.0'
 
 
 try:

allensdk/brain_observatory/behavior/behavior_ophys_session.py

@@ -516,11 +516,11 @@ def _get_roi_masks_by_cell_roi_id(self, cell_roi_ids=None):
         table.set_index("cell_roi_id", inplace=True)
         table = table.loc[cell_roi_ids, :]
 
-        full_image_shape = table.iloc[0]["image_mask"].shape
+        full_image_shape = table.iloc[0]["roi_mask"].shape
 
         output = np.zeros((len(cell_roi_ids), full_image_shape[0], full_image_shape[1]), dtype=np.uint8)
         for ii, (_, row) in enumerate(table.iterrows()):
-            output[ii, :, :] = _translate_roi_mask(row["image_mask"], int(row["y"]), int(row["x"]))
+            output[ii, :, :] = _translate_roi_mask(row["roi_mask"], int(row["y"]), int(row["x"]))
 
         # Pixel spacing and units of mask image will match either the
         # max or avg projection image of 2P movie.

allensdk/brain_observatory/behavior/session_apis/abcs/behavior_base.py

@@ -44,9 +44,14 @@ def get_rewards(self) -> pd.DataFrame:
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def get_running_data_df(self) -> pd.DataFrame:
+    def get_running_data_df(self, lowpass=True) -> pd.DataFrame:
         """Get running speed data.
 
+        Parameters
+        ----------
+        lowpass: bool
+            Whether to return running speed with low pass filter applied or without
+
         Returns
         -------
         pd.DataFrame

allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py

@@ -61,9 +61,10 @@ def save(self, session_object):
         # Add running data to NWB in-memory object:
         unit_dict = {'v_sig': 'V', 'v_in': 'V',
                      'speed': 'cm/s', 'timestamps': 's', 'dx': 'cm'}
-        nwb.add_running_data_df_to_nwbfile(nwbfile,
-                                           session_object.running_data_df,
-                                           unit_dict)
+        nwb.add_running_data_dfs_to_nwbfile(nwbfile,
+                                            session_object.running_data_df,
+                                            session_object.raw_running_data_df,
+                                            unit_dict)
 
         # Add stimulus template data to NWB in-memory object:
         for name, image_data in session_object.stimulus_templates.items():
@@ -148,9 +149,22 @@ def get_ophys_session_id(self) -> int:
     def get_eye_tracking(self) -> int:
         raise NotImplementedError()
 
-    def get_running_data_df(self, **kwargs) -> pd.DataFrame:
+    def get_running_data_df(self, lowpass=True) -> pd.DataFrame:
+        """
+        Gets the running data df
+        Parameters
+        ----------
+        lowpass: bool
+            Whether to return running speed with or without low pass filter applied
 
-        running_speed = self.get_running_speed()
+        Returns
+        -------
+            pd.DataFrame:
+                Dataframe containing various signals used to compute running
+                speed, and the filtered or unfiltered speed.
+        """
+
+        running_speed = self.get_running_speed(lowpass=lowpass)
 
         running_data_df = pd.DataFrame({'speed': running_speed.values},
                                        index=pd.Index(running_speed.timestamps,
@@ -263,6 +277,10 @@ def get_task_parameters(self) -> dict:
     def get_cell_specimen_table(self) -> pd.DataFrame:
         # NOTE: ROI masks are stored in full frame width and height arrays
         df = self.nwbfile.processing['ophys'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table'].to_dataframe()
+
+        # Because pynwb stores this field as "image_mask", it is renamed here
+        df = df.rename(columns={'image_mask': 'roi_mask'})
+
         df.index.rename('cell_roi_id', inplace=True)
         df['cell_specimen_id'] = [None if csid == -1 else csid for csid in df['cell_specimen_id'].values]
 

allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py

@@ -458,7 +458,7 @@ def get_raw_cell_specimen_table_dict(self) -> dict:
                 """.format(ophys_cell_seg_run_id)
         initial_cs_table = pd.read_sql(query, self.lims_db.get_connection())
         cell_specimen_table = initial_cs_table.rename(
-            columns={'id': 'cell_roi_id', 'mask_matrix': 'image_mask'})
+            columns={'id': 'cell_roi_id', 'mask_matrix': 'roi_mask'})
         cell_specimen_table.drop(['ophys_experiment_id',
                                   'ophys_cell_segmentation_run_id'],
                                  inplace=True, axis=1)

allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py

@@ -46,7 +46,7 @@ def get_cell_specimen_table(self):
         fov_height = self.get_field_of_view_shape()['height']
 
         # Convert cropped ROI masks to uncropped versions
-        image_mask_list = []
+        roi_mask_list = []
         for cell_roi_id, table_row in cell_specimen_table.iterrows():
             # Deserialize roi data into AllenSDK RoiMask object
             curr_roi = roi.RoiMask(image_w=fov_width, image_h=fov_height,
@@ -55,10 +55,10 @@ def get_cell_specimen_table(self):
             curr_roi.y = table_row['y']
             curr_roi.width = table_row['width']
             curr_roi.height = table_row['height']
-            curr_roi.mask = np.array(table_row['image_mask'])
-            image_mask_list.append(curr_roi.get_mask_plane().astype(np.bool))
+            curr_roi.mask = np.array(table_row['roi_mask'])
+            roi_mask_list.append(curr_roi.get_mask_plane().astype(np.bool))
 
-        cell_specimen_table['image_mask'] = image_mask_list
+        cell_specimen_table['roi_mask'] = roi_mask_list
         cell_specimen_table = cell_specimen_table[sorted(cell_specimen_table.columns)]
 
         cell_specimen_table.index.rename('cell_roi_id', inplace=True)
@@ -340,7 +340,7 @@ def get_corrected_fluorescence_traces(self):
         ophys_timestamps = self.get_ophys_timestamps()
 
         num_trace_timepoints = corrected_fluorescence_traces.shape[1]
-        assert num_trace_timepoints, ophys_timestamps.shape[0]
+        assert num_trace_timepoints == ophys_timestamps.shape[0]
         df = pd.DataFrame(
             {'corrected_fluorescence': list(corrected_fluorescence_traces)},
             index=pd.Index(cell_roi_id_list, name='cell_roi_id'))

allensdk/brain_observatory/behavior/write_nwb/_schemas.py

@@ -18,7 +18,7 @@ class CellSpecimenTable(RaisingSchema):
     height = Dict(String, Int, required=True)
     width = Dict(String, Int, required=True)
     mask_image_plane = Dict(String, Int, required=True)
-    image_mask = Dict(String, List(List(Boolean)), required=True)
+    roi_mask = Dict(String, List(List(Boolean)), required=True)
 
 
 

allensdk/brain_observatory/nwb/__init__.py

@@ -333,37 +333,48 @@ def add_running_speed_to_nwbfile(nwbfile, running_speed, name='speed', unit='cm/
         unit=unit
     )
 
-    running_mod = ProcessingModule('running', 'Running speed processing module')
-    nwbfile.add_processing_module(running_mod)
+    if 'running' in nwbfile.processing:
+        running_mod = nwbfile.processing['running']
+    else:
+        running_mod = ProcessingModule('running', 'Running speed processing module')
+        nwbfile.add_processing_module(running_mod)
 
     running_mod.add_data_interface(running_speed_series)
 
     return nwbfile
 
 
-def add_running_data_df_to_nwbfile(nwbfile, running_data_df, unit_dict, index_key='timestamps'):
-    ''' Adds running speed data to an NWBFile as timeseries in acquisition and processing
+def add_running_data_dfs_to_nwbfile(nwbfile, running_data_df, running_data_df_unfiltered, unit_dict):
+    """Adds both unfiltered (raw) and filtered running speed data to an NWBFile as timeseries in acquisition and processing
 
     Parameters
     ----------
     nwbfile : pynwb.NWBFile
-        File to which runnign speeds will be written
-    running_speed : pandas.DataFrame
-        Contains 'speed' and 'times', 'v_in', 'vsig', 'dx'
-    unit : str, optional
+        File to which running speeds will be written
+    running_data_df : pandas.DataFrame
+        Filtered running data
+        Contains 'speed', 'v_in', 'vsig', 'dx'
+        Note that 'v_in', 'vsig', 'dx' are expected to be the same as in running_data_df_unfiltered
+    running_data_df_unfiltered : pandas.DataFrame
+        Unfiltered (raw) Running data
+        Contains 'speed', 'v_in', 'vsig', 'dx'
+        Note that 'v_in', 'vsig', 'dx' are expected to be the same as in running_data_df
+    unit_dict : dict, optional
         SI units of running speed values
 
     Returns
     -------
     nwbfile : pynwb.NWBFile
 
-    '''
-    assert running_data_df.index.name == index_key
-
+    """
     running_speed = RunningSpeed(timestamps=running_data_df.index.values,
                                  values=running_data_df['speed'].values)
 
+    running_speed_unfiltered = RunningSpeed(timestamps=running_data_df_unfiltered.index.values,
+                                            values=running_data_df_unfiltered['speed'].values)
+
     add_running_speed_to_nwbfile(nwbfile, running_speed, name='speed', unit=unit_dict['speed'])
+    add_running_speed_to_nwbfile(nwbfile, running_speed_unfiltered, name='speed_unfiltered', unit=unit_dict['speed'])
 
     running_mod = nwbfile.processing['running']
     timestamps_ts = running_mod.get_data_interface('speed').timestamps
@@ -883,9 +894,9 @@ def add_cell_specimen_table(nwbfile: NWBFile,
         imaging_plane=imaging_plane)
 
     for col_name in cell_roi_table.columns:
-        # the columns 'image_mask', 'pixel_mask', and 'voxel_mask' are already defined
+        # the columns 'roi_mask', 'pixel_mask', and 'voxel_mask' are already defined
         # in the nwb.ophys::PlaneSegmentation Object
-        if col_name not in ['id', 'mask_matrix', 'image_mask', 'pixel_mask', 'voxel_mask']:
+        if col_name not in ['id', 'mask_matrix', 'roi_mask', 'pixel_mask', 'voxel_mask']:
             # This builds the columns with name of column and description of column
             # both equal to the column name in the cell_roi_table
             plane_segmentation.add_column(col_name,
@@ -895,13 +906,13 @@ def add_cell_specimen_table(nwbfile: NWBFile,
     # go through each roi and add it to the plan segmentation object
     for cell_roi_id, table_row in cell_roi_table.iterrows():
 
-        # NOTE: The 'image_mask' in this cell_roi_table has already been
+        # NOTE: The 'roi_mask' in this cell_roi_table has already been
         # processing by the function from 
         # allensdk.brain_observatory.behavior.session_apis.data_io.ophys_lims_api
         # get_cell_specimen_table() method. As a result, the ROI is stored in
         # an array that is the same shape as the FULL field of view of the
         # experiment (e.g. 512 x 512).
-        mask = table_row.pop('image_mask')
+        mask = table_row.pop('roi_mask')
 
         csid = table_row.pop('cell_specimen_id')
         table_row['cell_specimen_id'] = -1 if csid is None else csid

allensdk/brain_observatory/nwb/nwb_api.py

@@ -45,10 +45,23 @@ def from_path(cls, path, **kwargs):
 
         return cls(path=path, **kwargs)
 
-    def get_running_speed(self) -> RunningSpeed:
-
-        values = self.nwbfile.modules['running'].get_data_interface('speed').data[:]
-        timestamps = self.nwbfile.modules['running'].get_data_interface('speed').timestamps[:]
+    def get_running_speed(self, lowpass=True) -> RunningSpeed:
+        """
+        Gets the running speed
+        Parameters
+        ----------
+        lowpass: bool
+            Whether to return the running speed with lowpass filter applied or without
+
+        Returns
+        -------
+        RunningSpeed:
+            The running speed
+        """
+
+        interface_name = 'speed' if lowpass else 'speed_unfiltered'
+        values = self.nwbfile.modules['running'].get_data_interface(interface_name).data[:]
+        timestamps = self.nwbfile.modules['running'].get_data_interface(interface_name).timestamps[:]
 
         return RunningSpeed(
             timestamps=timestamps,

allensdk/internal/pipeline_modules/run_demixing.py

@@ -16,7 +16,12 @@
 import allensdk.core.json_utilities as ju
 import logging
 
-EXCLUDE_LABELS = ["union", "duplicate", "motion_border" ]
+EXCLUDE_LABELS = ["union", "duplicate", "motion_border",
+                  "decrosstalk_ghost",
+                  "decrosstalk_invalid_raw",
+                  "decrosstalk_invalid_raw_active",
+                  "decrosstalk_invalid_unmixed",
+                  "decrosstalk_invalid_unmixed_active" ]
 
 def debug(experiment_id, local=False):
     OUTPUT_DIRECTORY = "/data/informatics/CAM/demix"
@@ -34,26 +39,26 @@ def debug(experiment_id, local=False):
 """ % experiment_id)
 
     nrois = { roi['id']: dict(width=roi['width'],
-                              height=roi['height'], 
+                              height=roi['height'],
                               x=roi['x'],
                               y=roi['y'],
                               id=roi['id'],
                               valid=roi['valid_roi'],
                               mask=roi['mask_matrix'],
-                              exclusion_labels=[]) 
+                              exclusion_labels=[])
               for roi in rois }
 
     for exc_label in exc_labels:
         nrois[exc_label['id']]['exclusion_labels'].append(exc_label['exclusion_label'])
 
     movie_path_response = lu.query('''
         select wkf.filename, wkf.storage_directory from well_known_files wkf
-        join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id 
+        join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id
         where wkf.attachable_id = {} and wkf.attachable_type = 'OphysExperiment'
         and wkft.name = 'MotionCorrectedImageStack'
     '''.format(experiment_id))
     movie_h5_path = os.path.join(movie_path_response[0]['storage_directory'], movie_path_response[0]['filename'])
-        
+
     exp_dir = os.path.join(OUTPUT_DIRECTORY, str(experiment_id))
 
     input_data = {
@@ -62,9 +67,9 @@ def debug(experiment_id, local=False):
         "roi_masks": nrois.values(),
         "output_file": os.path.join(exp_dir, "demixed_traces.h5")
         }
- 
-    run_module(SCRIPT, 
-               input_data, 
+
+    run_module(SCRIPT,
+               input_data,
                exp_dir,
                sdk_path=SDK_PATH,
                pbs=dict(vmem=160,
@@ -99,8 +104,8 @@ def parse_input(data, exclude_labels):
         movie_shape = f["data"].shape[1:]
 
     with h5py.File(traces_h5, "r") as f:
-        traces = f["data"].value
-        trace_ids = [ int(rid) for rid in f["roi_names"].value ]
+        traces = f["data"][()]
+        trace_ids = [ int(rid) for rid in f["roi_names"][()] ]
 
     rois = get_path(data, "roi_masks", False)
     masks = None
@@ -135,7 +140,7 @@ def main():
     logging.debug("reading input")
 
     traces, masks, valid, trace_ids, movie_h5, output_h5 = parse_input(data, mod.args.exclude_labels)
-    
+
     logging.debug("excluded masks: %s", str(zip(np.where(~valid)[0], trace_ids[~valid])))
     output_dir = os.path.dirname(output_h5)
     plot_dir = os.path.join(output_dir, "demix_plots")
@@ -145,7 +150,7 @@ def main():
 
     logging.debug("reading movie")
     with h5py.File(movie_h5, 'r') as f:
-        movie = f['data'].value
+        movie = f['data'][()]
 
     # only demix non-union, non-duplicate ROIs
     valid_idxs = np.where(valid)
@@ -154,27 +159,27 @@ def main():
 
     logging.debug("demixing")
     demixed_traces, drop_frames = demixer.demix_time_dep_masks(demix_traces, movie, demix_masks)
-    
-    nt_inds = demixer.plot_negative_transients(demix_traces, 
-                                               demixed_traces, 
+
+    nt_inds = demixer.plot_negative_transients(demix_traces,
+                                               demixed_traces,
                                                valid[valid_idxs],
                                                demix_masks,
                                                trace_ids[valid_idxs],
                                                plot_dir)
 
     logging.debug("rois with negative transients: %s", str(trace_ids[valid_idxs][nt_inds]))
 
-    nb_inds = demixer.plot_negative_baselines(demix_traces, 
-                                              demixed_traces, 
+    nb_inds = demixer.plot_negative_baselines(demix_traces,
+                                              demixed_traces,
                                               demix_masks,
                                               trace_ids[valid_idxs],
                                               plot_dir)
 
-    # negative baseline rois (and those that overlap with them) become nans 
+    # negative baseline rois (and those that overlap with them) become nans
     logging.debug("rois with negative baselines (or overlap with them): %s", str(trace_ids[valid_idxs][nb_inds]))
     demixed_traces[nb_inds, :] = np.nan
 
-    logging.info("Saving output")    
+    logging.info("Saving output")
     out_traces = np.zeros(traces.shape, dtype=demix_traces.dtype)
     out_traces[:] = np.nan
     out_traces[valid_idxs] = demixed_traces

allensdk/test/brain_observatory/behavior/conftest.py

@@ -196,7 +196,7 @@ def cell_specimen_table():
                          'max_correction_right': [1., 1.],
                          'mask_image_plane': [1, 1],
                          'ophys_cell_segmentation_run_id': [1, 1],
-                         'image_mask': [np.array([[True, True],
+                         'roi_mask': [np.array([[True, True],
                                                   [True, False]]),
                                         np.array([[True, True],
                                                   [False, True]])]},