Feat 20 remove ephys

.github/workflows/tag_and_publish.yml

@@ -14,10 +14,10 @@ jobs:
         ref: ${{ env.DEFAULT_BRANCH }}
         fetch-depth: 0
         token: ${{ secrets.SERVICE_TOKEN }}
-    - name: Set up Python 3.10
+    - name: Set up Python 3.8
       uses: actions/setup-python@v3
       with:
-        python-version: '3.10'
+        python-version: '3.8'
     - name: Install dependencies
       run: | 
         python -m pip install -e .[dev] --no-cache-dir
@@ -64,55 +64,22 @@ jobs:
     secrets:
       SERVICE_TOKEN: ${{ secrets.SERVICE_TOKEN }}
   publish:
-    runs-on: ubuntu-latest
-    env:
-      GO_VERSION: '1.14.12'
-      GO_OS: 'linux'
-      GO_ARCH: 'amd64'
-      SINGULARITY_VERSION: '3.7.0'
     needs: tag
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: Pull latest changes
         run: git pull origin main
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
       - name: Install dependencies
         run: |
-          sudo apt-get update && sudo apt-get install -y \
-          build-essential \
-          libssl-dev \
-          uuid-dev \
-          libgpgme11-dev \
-          squashfs-tools \
-          libseccomp-dev \
-          wget \
-          pkg-config \
-          procps
-      - name: Install GO
-        run: |
-          wget https://dl.google.com/go/go$GO_VERSION.$GO_OS-$GO_ARCH.tar.gz
-          sudo tar -C /usr/local -xzvf go$GO_VERSION.$GO_OS-$GO_ARCH.tar.gz
-          sudo rm go$GO_VERSION.$GO_OS-$GO_ARCH.tar.gz
-          echo 'export PATH=$PATH:/usr/local/go/bin' | sudo tee -a /etc/profile
-      - name: Install Singularity
-        run: |
-          cd ..
-          wget https://github.com/hpcng/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-${SINGULARITY_VERSION}.tar.gz
-          sudo tar -xzf singularity-${SINGULARITY_VERSION}.tar.gz
-          sudo rm singularity-${SINGULARITY_VERSION}.tar.gz
-          echo "Finished installing binaries"
-          export PATH=$PATH:/usr/local/go/bin
-          cd singularity
-          sudo ./mconfig --without-suid
-          sudo make -C ./builddir
-          sudo make -C ./builddir install
-      - name: Build sif container
-        run: |
-          ls /
-          ls
-          mkdir build
-          sudo singularity build build/container.sif scripts/singularity_build.def
-      - name: Login and Deploy Container
-        run: |
-          echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ github.actor }} --password-stdin oras://ghcr.io
-          singularity push build/container.sif oras://ghcr.io/${GITHUB_REPOSITORY}:${{ needs.tag.outputs.new_version }}
-          singularity push build/container.sif oras://ghcr.io/${GITHUB_REPOSITORY}:latest
+          pip install --upgrade setuptools wheel twine build
+          python -m build
+          twine check dist/*
+      - name: Publish on PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.AIND_PYPI_TOKEN }}

.github/workflows/test_and_lint.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.9', '3.10', '3.11' ]
+        python-version: [ '3.8', '3.9', '3.10', '3.11' ]
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}

README.md

@@ -9,43 +9,41 @@
 
 ## Usage
 
-There are 4 main ways to run a data transformation job:
-- from a python script
-- from the command line passing in the settings as a json string
-- from the command line pointing to a config file
-- from the command line with env vars
-
-Assuming `output_dir` exists:
-
-### From python
+Please import this package and extend the abstract base class to define a new transformation job
 ```python
-from aind_data_transformation.ephys.ephys_job import EphysJobSettings, EphysCompressionJob
-from pathlib import Path
-
-input_source = Path("./tests/resources/v0.6.x_neuropixels_multiexp_multistream")
-output_dir = Path("output_dir")
-
-job_settings = EphysJobSettings(input_source=input_source, output_directory=output_dir)
-job = EphysCompressionJob(job_settings=job_settings)
+from aind_data_transformation.core import (
+    BasicJobSettings,
+    GenericEtl,
+    JobResponse,
+)
+
+# An example JobSettings
+class NewTransformJobSettings(BasicJobSettings):
+  # Add extra fields needed, for example, a random seed
+  random_seed: Optional[int] = 0
+
+# An example EtlJob
+class NewTransformJob(GenericEtl[NewTransformJobSettings]):
+
+    # This method needs to be defined
+    def run_job(self) -> JobResponse:
+        """
+        Main public method to run the transformation job
+        Returns
+        -------
+        JobResponse
+          Information about the job that can be used for metadata downstream.
+
+        """
+        job_start_time = datetime.now()
+        # Do something here
+        job_end_time = datetime.now()
+        return JobResponse(
+            status_code=200,
+            message=f"Job finished in: {job_end_time-job_start_time}",
+            data=None,
+        )
 
-response = job.run_job()
-```
-
-### From the command line passing in settings as a json str
-```bash
-python -m aind_data_transformation.ephys.ephys_job --job-settings '{"input_source":"./tests/resources/v0.6.x_neuropixels_multiexp_multistream","output_directory":"output_dir"}'
-```
-
-### From the command line passing in settings via a config file
-```bash
-python -m aind_data_transformation.ephys.ephys_job --config-file configs.json
-```
-
-### From the command line passing in settings via environment variables
-```bash
-export TRANSFORMATION_JOB_INPUT_SOURCE="./tests/resources/v0.6.x_neuropixels_multiexp_multistream"
-export TRANSFORMATION_JOB_OUTPUT_DIRECTORY="output_dir"
-python -m aind_data_transformation.ephys.ephys_job
 ```
 
 

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "aind-data-transformation"
 description = "Generated from aind-library-template"
 license = {text = "MIT"}
-requires-python = ">=3.9"
+requires-python = ">=3.8"
 authors = [
     {name = "Allen Institute for Neural Dynamics"}
 ]
@@ -22,15 +22,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-ephys = [
-    'spikeinterface[full]>=0.100.5',
-    'probeinterface==0.2.21',
-    'wavpack-numcodecs>=0.1.3,<=0.1.5',
-    'zarr==2.16.1',
-    'numcodecs==0.11.0'
-]
 dev = [
-    'aind-data-transformation[ephys]',
     'black',
     'coverage',
     'flake8',
@@ -48,7 +40,7 @@ version = {attr = "aind_data_transformation.__version__"}
 
 [tool.black]
 line-length = 79
-target_version = ['py39']
+target_version = ['py38']
 exclude = '''
 
 (