Merge branch 'rc/2.14.0'

AllenInstitute · Dec 12, 2022 · 307296b · 307296b
2 parents f69f064 + f38fc25
commit 307296b
Show file tree

Hide file tree

Showing 187 changed files with 328,342 additions and 105,292 deletions.
diff --git a/.github/workflows/github-actions-ci.yml b/.github/workflows/github-actions-ci.yml
@@ -44,7 +44,7 @@ jobs:
     strategy:
       matrix:
         os: ["macos-latest", "windows-latest", "ubuntu-latest"]
-        python-version: ["3.6", "3.7", "3.8"]
+        python-version: ["3.7", "3.8", "3.9"]
       fail-fast: false
     defaults:
       run:
@@ -71,7 +71,7 @@ jobs:
     runs-on: ["self-hosted"]
     strategy:
       matrix:
-        image: ["allensdk_local_py37:latest"]
+        image: ["allensdk_local_py38:latest"]
     steps:
       - uses: actions/checkout@v2
       - name: run test in docker
@@ -87,11 +87,12 @@ jobs:
                   /bin/bash -c "pip install -r requirements.txt; \
                                 pip install -r test_requirements.txt; \
                                 python -m pytest \
+                                    --log-level=INFO \
                                     --capture=no \
                                     --cov=allensdk \
                                     --cov-config coveragerc \
                                     --cov-report html \
                                     --junitxml=test-reports/test.xml \
                                     --boxed \
-                                    --numprocesses 4 \
+                                    --numprocesses auto \
                                     --durations=0"
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -3,14 +3,15 @@ on:
   schedule:
     - cron: '05 6 * * *'
 
+
 jobs:
   onprem:
     name: python ${{ matrix.image }} on-prem test
     runs-on: ["self-hosted"]
     strategy:
       matrix:
         image: ["allensdk_local_py37:latest"]
-        branch: ["master", "rc/2.16.1"]
+        branch: ["master", "rc/**"]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -29,11 +30,12 @@ jobs:
                                 pip install -r test_requirements.txt; \
                                 pip install -e .; \
                                 python -m pytest \
+                                    --log-level=INFO \
                                     --capture=no \
                                     --cov=allensdk \
                                     --cov-config coveragerc \
                                     --cov-report html \
                                     --junitxml=test-reports/test.xml \
                                     --boxed \
-                                    --numprocesses 4 \
-                                    --durations=0"
+                                    --numprocesses auto \
+                                    --durations=0"
diff --git a/.github/workflows/notebook_runner.yml b/.github/workflows/notebook_runner.yml
@@ -0,0 +1,37 @@
+name: Notebooks
+on:
+  pull_request:
+    branches:
+      - master
+jobs:
+  run_notebooks:
+    name: Notebook runner
+    strategy:
+      matrix:
+        os: ["ubuntu-latest"]
+        python-version: ["3.8"]
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: |
+          pip install -r requirements.txt
+          pip install -r dev_requirements.txt
+          pip install .
+          ipython kernel install --name "python3" --user
+      - name: Run notebooks
+        run: |
+          python -m allensdk.internal.notebooks.execute_notebooks \
+            --notebooks_dir doc_template/examples_root/examples/nb \
+            --skip_notebooks behavior_ophys_session.ipynb
+      - name: Commit and push updated notebooks
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: Update notebooks
+          push_options: --force
+          file_pattern: '*.ipynb'
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,13 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.14.0] = 2022-12-12
+- Support for updated vbn release containing probe, lfp and behavior only data.
+- Updates to Ophys data in anticipation of a forthcoming updated data release
+- Various notebook updates
+- Python 3.9 support
+- Various bug fixes and quality of life improvements.
+
 ## [2.13.6] = 2022-08-04
 - bugfix when accessing stimulus presentations table for vcn data
 - updates vbn notebooks

diff --git a/allensdk/__init__.py b/allensdk/__init__.py
@@ -35,7 +35,7 @@
 #
 import logging
 
-__version__ = '2.13.9'
+__version__ = '2.14.0'
 
 
 try:

diff --git a/allensdk/api/queries/donors_queries.py b/allensdk/api/queries/donors_queries.py
diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py b/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py
@@ -2,44 +2,22 @@
 import pandas as pd
 from pynwb import NWBFile
 
-from allensdk.brain_observatory.behavior.behavior_session import (
-    BehaviorSession)
+from allensdk.brain_observatory.behavior.behavior_session import BehaviorSession  # NOQA
 from allensdk.brain_observatory.behavior.data_files import SyncFile
-from allensdk.brain_observatory.behavior.data_files\
-    .rigid_motion_transform_file import \
-    RigidMotionTransformFile
-from allensdk.brain_observatory.behavior.data_objects import \
-    BehaviorSessionId
-from allensdk.brain_observatory.behavior.data_objects.cell_specimens \
-    .cell_specimens import \
-    CellSpecimens, EventsParams
-from allensdk.brain_observatory.behavior.data_objects.metadata \
-    .behavior_metadata.date_of_acquisition import \
-    DateOfAcquisitionOphys, DateOfAcquisition
-from allensdk.brain_observatory.behavior.data_objects.metadata\
-    .behavior_ophys_metadata import \
-    BehaviorOphysMetadata
-from allensdk.brain_observatory.behavior.data_objects.metadata\
-    .ophys_experiment_metadata.multi_plane_metadata.imaging_plane_group \
-    import \
-    ImagingPlaneGroup
-from allensdk.brain_observatory.behavior.data_objects.metadata\
-    .ophys_experiment_metadata.multi_plane_metadata.multi_plane_metadata \
-    import \
-    MultiplaneMetadata
-from allensdk.brain_observatory.behavior.data_objects.motion_correction \
-    import \
-    MotionCorrection
-from allensdk.brain_observatory.behavior.data_objects.projections import \
-    Projections
-from allensdk.brain_observatory.behavior.data_objects.stimuli.util import \
-    calculate_monitor_delay
-from allensdk.brain_observatory.behavior.data_objects.timestamps \
-    .ophys_timestamps import \
-    OphysTimestamps, OphysTimestampsMultiplane
+from allensdk.brain_observatory.behavior.data_files.rigid_motion_transform_file import RigidMotionTransformFile  # NOQA
+from allensdk.brain_observatory.behavior.data_objects import BehaviorSessionId
+from allensdk.brain_observatory.behavior.data_objects.cell_specimens.cell_specimens import CellSpecimens, EventsParams  # NOQA
+from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_metadata.date_of_acquisition import DateOfAcquisition, DateOfAcquisitionOphys  # NOQA
+from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_ophys_metadata import BehaviorOphysMetadata  # NOQA
+from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.multi_plane_metadata.imaging_plane_group import ImagingPlaneGroup  # NOQA
+from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.multi_plane_metadata.multi_plane_metadata import MultiplaneMetadata  # NOQA
+from allensdk.brain_observatory.behavior.data_objects.motion_correction import MotionCorrection  # NOQA
+from allensdk.brain_observatory.behavior.data_objects.projections import Projections  # NOQA
+from allensdk.brain_observatory.behavior.data_objects.stimuli.util import calculate_monitor_delay  # NOQA
+from allensdk.brain_observatory.behavior.data_objects.timestamps.ophys_timestamps import OphysTimestamps, OphysTimestampsMultiplane  # NOQA
+from allensdk.brain_observatory.behavior.image_api import Image
 from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP
 from allensdk.deprecated import legacy
-from allensdk.brain_observatory.behavior.image_api import Image
 from allensdk.internal.api import db_connection_creator
 
 
@@ -100,8 +78,7 @@ def from_lims(cls,
                   eye_tracking_dilation_frames: int = 2,
                   events_filter_scale_seconds: float = 2.0/31.0,
                   events_filter_n_time_steps: int = 20,
-                  exclude_invalid_rois=True,
-                  skip_eye_tracking=False) -> \
+                  exclude_invalid_rois=True) -> \
             "BehaviorOphysExperiment":
         """
         Parameters
@@ -117,8 +94,6 @@ def from_lims(cls,
             See `BehaviorOphysExperiment.from_nwb`
         exclude_invalid_rois
             Whether to exclude invalid rois
-        skip_eye_tracking
-            Used to skip returning eye tracking data
         """
         def _is_multi_plane_session():
             imaging_plane_group_meta = ImagingPlaneGroup.from_lims(
@@ -162,7 +137,6 @@ def _get_motion_correction():
             sync_file=sync_file,
             monitor_delay=monitor_delay,
             date_of_acquisition=date_of_acquisition,
-            skip_eye_tracking=skip_eye_tracking,
             eye_tracking_z_threshold=eye_tracking_z_threshold,
             eye_tracking_dilation_frames=eye_tracking_dilation_frames
         )
@@ -275,8 +249,7 @@ def from_json(cls,
                   eye_tracking_dilation_frames: int = 2,
                   events_filter_scale_seconds: float = 2.0/31.0,
                   events_filter_n_time_steps: int = 20,
-                  exclude_invalid_rois=True,
-                  skip_eye_tracking=False) -> \
+                  exclude_invalid_rois=True) -> \
             "BehaviorOphysExperiment":
         """
 
@@ -293,8 +266,6 @@ def from_json(cls,
             See `BehaviorOphysExperiment.from_nwb`
         exclude_invalid_rois
             Whether to exclude invalid rois
-        skip_eye_tracking
-            Used to skip returning eye tracking data
 
         """
         def _is_multi_plane_session():
@@ -426,13 +397,15 @@ def metadata(self):
             'emission_lambda': self._cell_specimens.meta.emission_lambda,
             'excitation_lambda':
                 self._cell_specimens.meta.imaging_plane.excitation_lambda,
-            'experiment_container_id':
-                self._metadata.ophys_metadata.experiment_container_id,
+            'ophys_container_id':
+                self._metadata.ophys_metadata.ophys_container_id,
             'field_of_view_height':
                 self._metadata.ophys_metadata.field_of_view_shape.height,
             'field_of_view_width':
                 self._metadata.ophys_metadata.field_of_view_shape.width,
             'imaging_depth': self._metadata.ophys_metadata.imaging_depth,
+            'targeted_imaging_depth':
+                self._metadata.ophys_metadata.targeted_imaging_depth,
             'imaging_plane_group':
                 self._metadata.ophys_metadata.imaging_plane_group
                 if isinstance(self._metadata.ophys_metadata,
@@ -602,10 +575,60 @@ def corrected_fluorescence_traces(self) -> pd.DataFrame:
                     (assigned before cell matching)
                 corrected_fluorescence: (list of float)
                     fluorescence values (arbitrary units)
-
+                RMSE: (float)
+                    error values (arbitrary units)
+                r:
+                    r values (arbitrary units)
         """
         return self._cell_specimens.corrected_fluorescence_traces
 
+    @property
+    def demixed_traces(self) -> pd.DataFrame:
+        """Demixed traces are traces that are demixed from overlapping ROIs.
+        Sampling rate can be found in metadata ‘ophys_frame_rate’
+
+        Returns
+        -------
+        pd.DataFrame
+            Dataframe that contains the corrected fluorescence traces
+            for all valid cells.
+
+            dataframe columns:
+                cell_specimen_id [index]: (int)
+                    unified id of segmented cell across experiments
+                    (assigned after cell matching)
+                cell_roi_id: (int)
+                    experiment specific id of segmented roi
+                    (assigned before cell matching)
+                demixed_trace: (list of float)
+                    fluorescence values (arbitrary units)
+        """
+        return self._cell_specimens.demixed_traces
+
+    @property
+    def neuropil_traces(self) -> pd.DataFrame:
+        """neuropil traces are the fluorescent signal measured from the
+        neuropil_masks. Sampling rate can be found in metadata
+        ‘ophys_frame_rate’
+
+        Returns
+        -------
+        pd.DataFrame
+            Dataframe that contains the corrected fluorescence traces
+            for all valid cells.
+
+            dataframe columns:
+                cell_specimen_id [index]: (int)
+                    unified id of segmented cell across experiments
+                    (assigned after cell matching)
+                cell_roi_id: (int)
+                    experiment specific id of segmented roi
+                    (assigned before cell matching)
+                neuropil_trace: (list of float)
+                    fluorescence values (arbitrary units)
+        """
+        return self._cell_specimens.demixed_traces
+
     @property
     def motion_correction(self) -> pd.DataFrame:
         """a dataframe containing the x and y offsets applied during