Merge pull request #305 from ICB-DCM/develop

Release 0.0.13

.gitignore

@@ -103,3 +103,4 @@ amici_models/*
 *.txt
 test/doc/example/tmp/benchmark-models/
 test/amici_models/*
+*.hdf5

.rtd_pip_reqs.txt

@@ -1,2 +1,3 @@
+ipython
 nbsphinx
 sphinx_rtd_theme

doc/api_engine.rst

@@ -2,3 +2,4 @@
    :members:
    :inherited-members:
    :special-members:
+   :imported-members:

doc/api_logging.rst

@@ -0,0 +1,5 @@
+.. automodule:: pypesto.logging
+   :members:
+   :inherited-members:
+   :special-members:
+   :imported-members:

doc/api_objective.rst

@@ -2,3 +2,4 @@
    :members:
    :inherited-members:
    :special-members:
+   :imported-members:

doc/api_optimize.rst

@@ -2,3 +2,4 @@
    :members:
    :inherited-members:
    :special-members:
+   :imported-members:

doc/api_sample.rst → doc/api_petab.rst

@@ -1,4 +1,5 @@
-.. automodule:: pypesto.sample
+.. automodule:: pypesto.petab
    :members:
    :inherited-members:
    :special-members:
+   :imported-members:

doc/api_problem.rst

@@ -2,3 +2,4 @@
    :members:
    :inherited-members:
    :special-members:
+   :imported-members:

doc/api_profile.rst

@@ -2,3 +2,4 @@
    :members:
    :inherited-members:
    :special-members:
+   :imported-members:

doc/api_result.rst

@@ -2,3 +2,4 @@
    :members:
    :inherited-members:
    :special-members:
+   :imported-members:

doc/api_sampling.rst

@@ -0,0 +1,5 @@
+.. automodule:: pypesto.sampling
+   :members:
+   :inherited-members:
+   :special-members:
+   :imported-members:

doc/api_startpoint.rst

@@ -2,3 +2,4 @@
    :members:
    :inherited-members:
    :special-members:
+   :imported-members:

doc/api_storage.rst

@@ -0,0 +1,5 @@
+.. automodule:: pypesto.storage
+   :members:
+   :inherited-members:
+   :special-members:
+   :imported-members:

doc/api_visualize.rst

@@ -2,3 +2,4 @@
    :members:
    :inherited-members:
    :special-members:
+   :imported-members:

doc/conf.py

@@ -33,13 +33,15 @@
 # ones.
 extensions = ['sphinx.ext.autodoc',
               'sphinx.ext.napoleon',
+              'IPython.sphinxext.ipython_console_highlighting',
               'nbsphinx']
 
 # default autodoc options
 # list for special-members seems not to be possible before 1.8
 autodoc_default_flags = ['members',
                          'undoc-members',
-                         'show-inheritance']
+                         'show-inheritance',
+                         'imported-members']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

doc/example/conversion_reaction/conditions.tsv

@@ -0,0 +1,2 @@
+conditionId
+c0

doc/example/conversion_reaction/conversion_reaction.yaml

@@ -0,0 +1,11 @@
+format_version: 1
+parameter_file: parameters.tsv
+problems:
+- condition_files:
+  - conditions.tsv
+  measurement_files:
+  - measurements.tsv
+  observable_files:
+  - observables.tsv
+  sbml_files:
+  - model_conversion_reaction.xml

doc/example/conversion_reaction/create.py

@@ -0,0 +1,73 @@
+from petab.C import *
+import petab
+
+import pandas as pd
+import numpy as np
+
+a0 = 1
+b0 = 0
+k1 = 0.8
+k2 = 0.6
+
+
+def analytical_a(t, a0=a0, b0=b0, k1=k1, k2=k2):
+    return k2 * (a0 + b0) / (k1 + k2) \
+           + (a0 - k2 * (a0 + b0) / (k1 + k2)) * np.exp(-(k1 + k2) * t)
+
+
+# problem --------------------------------------------------------------------
+
+condition_df = pd.DataFrame(data={
+    CONDITION_ID: ['c0'],
+}).set_index([CONDITION_ID])
+
+times = np.linspace(0, 3, 10)
+nt = len(times)
+simulations = [analytical_a(t, 1, 0, 0.8, 0.6)
+               for t in times]
+sigma = 0.02
+measurements = simulations + sigma * np.random.randn(nt)
+
+measurement_df = pd.DataFrame(data={
+    OBSERVABLE_ID: ['obs_a'] * nt,
+    SIMULATION_CONDITION_ID: ['c0'] * nt,
+    TIME: times,
+    MEASUREMENT: measurements
+})
+
+observable_df = pd.DataFrame(data={
+    OBSERVABLE_ID: ['obs_a'],
+    OBSERVABLE_FORMULA: ['A'],
+    NOISE_FORMULA: [sigma]
+}).set_index([OBSERVABLE_ID])
+
+parameter_df = pd.DataFrame(data={
+    PARAMETER_ID: ['k1', 'k2'],
+    PARAMETER_SCALE: [LOG] * 2,
+    LOWER_BOUND: [1e-5] * 2,
+    UPPER_BOUND: [1e5] * 2,
+    NOMINAL_VALUE: [k1, k2],
+    ESTIMATE: [1, 1],
+}).set_index(PARAMETER_ID)
+
+
+petab.write_condition_df(condition_df, "conditions.tsv")
+petab.write_measurement_df(measurement_df, "measurements.tsv")
+petab.write_observable_df(observable_df, "observables.tsv")
+petab.write_parameter_df(parameter_df, "parameters.tsv")
+
+yaml_config = {
+    FORMAT_VERSION: 1,
+    PARAMETER_FILE: "parameters.tsv",
+    PROBLEMS: [{
+        SBML_FILES: ["model_conversion_reaction.xml"],
+        CONDITION_FILES: ["conditions.tsv"],
+        MEASUREMENT_FILES: ["measurements.tsv"],
+        OBSERVABLE_FILES: ["observables.tsv"]
+    }]
+}
+petab.write_yaml(yaml_config, "conversion_reaction.yaml")
+
+# validate written PEtab files
+problem = petab.Problem.from_yaml("conversion_reaction.yaml")
+petab.lint_problem(problem)

doc/example/conversion_reaction/measurements.tsv

@@ -0,0 +1,11 @@
+observableId	simulationConditionId	time	measurement
+obs_a	c0	0.0	1.0321025178287548
+obs_a	c0	0.3333333333333333	0.8009487310753414
+obs_a	c0	0.6666666666666666	0.6522988284518845
+obs_a	c0	1.0	0.5468869037277241
+obs_a	c0	1.3333333333333333	0.5338962162237411
+obs_a	c0	1.6666666666666665	0.48794403101997796
+obs_a	c0	2.0	0.44706262564427257
+obs_a	c0	2.333333333333333	0.4187284503596733
+obs_a	c0	2.6666666666666665	0.4586806097362004
+obs_a	c0	3.0	0.4106899489905058

doc/example/conversion_reaction/observables.tsv

@@ -0,0 +1,2 @@
+observableId	observableFormula	noiseFormula
+obs_a	A	0.02

doc/example/conversion_reaction/parameters.tsv

@@ -0,0 +1,3 @@
+parameterId	parameterScale	lowerBound	upperBound	nominalValue	estimate
+k1	log	1e-05	100000.0	0.8	1
+k2	log	1e-05	100000.0	0.6	1

doc/examples.rst

@@ -12,6 +12,7 @@ The following examples cover typical use cases and should help get a better idea
    example/boehm_JProteomeRes2014.ipynb
    example/petab_import.ipynb
    example/hdf5_storage_result.ipynb
+   example/sampler_study.ipynb
 
 Download the examples as notebooks
 ----------------------------------
@@ -22,6 +23,7 @@ Download the examples as notebooks
 * :download:`Boehm model <example/boehm_JProteomeRes2014.ipynb>`
 * :download:`Petab import <example/petab_import.ipynb>`
 * :download:`HDF5 storage <example/hdf5_storage_result.ipynb>`
+* :download:`Sampler study <example/sampler_study.ipynb>`
 
 .. Note::
    Some of the notebooks have extra dependencies.

doc/index.rst

@@ -49,13 +49,16 @@ Welcome to pyPESTO's documentation!
 
    api_objective
    api_problem
+   api_petab
    api_optimize
    api_profile
-   api_sample
+   api_sampling
+   api_visualize
    api_result
    api_engine
-   api_visualize
    api_startpoint
+   api_storage
+   api_logging
 
 .. toctree::
    :maxdepth: 2

doc/releasenotes.rst

@@ -6,6 +6,32 @@ Release notes
 ..........
 
 
+0.0.13 (2020-05-03)
+-------------------
+
+* Tidy up and speed up tests (#265 and others).
+* Basic self-implemented Adaptive Metropolis and Adaptive Parallel Tempering
+  sampling routines (#268).
+* Fix namespace sample -> sampling (#275).
+* Fix covariance matrix regularization (#275).
+* Fix circular dependency `PetabImporter` - `PetabAmiciObjective` via
+  `AmiciObjectBuilder`, `PetabAmiciObjective` becomes obsolete (#274).
+* Define `AmiciCalculator` to separate the AMICI call logic (required for
+  hierarchical optimization) (#277).
+* Define initialize function for resetting steady states in `AmiciObjective`
+  (#281).
+* Fix scipy least squares options (#283).
+* Allow failed starts by default (#280).
+* Always copy parameter vector in objective to avoid side effects (#291).
+* Add Dockerfile (#288).
+* Fix header names in CSV history (#299).
+
+Documentation:
+
+* Use imported members in autodoc (#270).
+* Enable python syntax highlighting in notebooks (#271).
+
+
 0.0.12 (2020-04-06)
 -------------------
 

docker/Dockerfile

@@ -0,0 +1,36 @@
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Berlin
+
+RUN apt update \
+    && apt-get install -y \
+    g++ \
+    cmake \
+    libatlas-base-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    swig \
+    git\
+    libhdf5-serial-dev\
+    && ln -sf /usr/bin/swig4.0 /usr/bin/swig
+
+RUN pip3 install python-libsbml>=5.17.0
+
+COPY amici.tar.gz /tmp
+
+ENV AMICI_CXXFLAGS -fopenmp
+ENV AMICI_LDFLAGS -fopenmp
+
+RUN pip3 install -U --upgrade pip wheel \
+    && mkdir -p /tmp/amici/ \
+    && cd /tmp/amici \
+    && tar -xzf ../amici.tar.gz \
+    && cd /tmp/amici/python/sdist \
+    && python3 setup.py -v sdist \
+    && pip3 install -v $(ls -t /tmp/amici/python/sdist/dist/amici-*.tar.gz | head -1)[petab,pysb] \
+    && rm -rf /tmp && mkdir /tmp
+
+# RUN pip3 install git+https://github.com/ICB-DCM/pyPESTO.git@develop#egg=pypesto
+RUN pip3 install pyPESTO jupyter pyswarm dlib

docker/README.md

@@ -0,0 +1,26 @@
+# AMICI & pyPESTO with Docker
+
+## Create image
+
+In the AMICI base directory run:
+
+```bash
+git archive -o <path to pypesto base directory>/docker/amici.tar.gz --format=tar.gz HEAD
+cd <path to pypesto base directory>/docker && docker build -t $USER/amici_pypesto:latest .
+```
+
+To install pyPESTO from a particular branch, e.g. develop, use th following 
+line in the Dockerfile
+
+```
+RUN pip3 install git+https://github.com/ICB-DCM/pyPESTO.git@develop#egg=pypesto 
+```
+
+environment file can be used with `--set-env` option of `ch-run` command. From 
+charliecloud documentation: 
+
+"
+The purpose of `--set-env=FILE` is to set environment variables that cannot be 
+inherited from the host shell, e.g. Dockerfile ENV directives or other 
+build-time configuration
+"

docker/environment

@@ -0,0 +1,2 @@
+AMICI_CXXFLAGS=-fopenmp
+AMICI_LDFLAGS=-fopenmp

pypesto/__init__.py

@@ -16,9 +16,10 @@
     Hdf5History,
     OptimizerHistory,
     Objective,
-    AmiciObjective,
-    PetabImporter)
+    AmiciObjective)
 from .problem import Problem
+from .petab import (
+    PetabImporter)
 from . import startpoint
 from .result import (
     Result,
@@ -37,6 +38,15 @@
     parameter_profile,
     ProfileOptions,
     ProfilerResult)
+from .sampling import (
+    sample,
+    Sampler,
+    InternalSampler,
+    MetropolisSampler,
+    AdaptiveMetropolisSampler,
+    ParallelTemperingSampler,
+    AdaptiveParallelTemperingSampler,
+    McmcPtResult)
 from .engine import (
     SingleCoreEngine,
     MultiThreadEngine,

pypesto/logging.py

@@ -1,3 +1,10 @@
+"""
+Logging
+=======
+
+Logging convenience functions.
+"""
+
 import logging
 
 

pypesto/objective/__init__.py

@@ -1,11 +1,11 @@
 """
 Objective
 =========
-
 """
 
 from .objective import Objective
-from .amici_objective import AmiciObjective
+from .amici_calculator import AmiciCalculator
+from .amici_objective import AmiciObjective, AmiciObjectBuilder
 from .aggregated import AggregatedObjective
 from .util import res_to_chi2, sres_to_schi2
 from .history import (
@@ -17,9 +17,3 @@
     Hdf5History,
     OptimizerHistory)
 from . import constants
-
-# PEtab is an optional dependency
-try:
-    from .petab_import import PetabImporter
-except ModuleNotFoundError:
-    PetabImporter = None