Merge pull request ICB-DCM#1298 from ICB-DCM/develop

Release v0.4.2

.github/CODEOWNERS

@@ -1,27 +1,31 @@
 # see https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
 
-# default owners
-* @ICB-DCM/pypesto-maintainers
+# default owners = active maintainers
+* @Doresic @PaulJonasJost @m-philipps
 
-/doc/example/censored.ipynb @Doresic
+# Examples
+/doc/example/censored_data.ipynb @Doresic
 /doc/example/hdf5_storage.ipynb @PaulJonasJost
-/doc/example/hierarchical.ipynb @dilpath @dweindl
 /doc/example/julia.ipynb @PaulJonasJost
 /doc/example/model_selection.ipynb @dilpath
-/doc/example/nonlinear_monotone.ipynb @Doresic
-/doc/example/ordinal.ipynb @Doresic
+/doc/example/ordinal_data.ipynb @Doresic
 /doc/example/petab_import.ipynb @dweindl @FFroehlich
+/doc/example/relative_data.ipynb @dilpath @dweindl
 /doc/example/sampler_study.ipynb @dilpath
 /doc/example/sampling_diagnostics.ipynb @dilpath
+/doc/example/semiquantitative_data.ipynb @Doresic
 /doc/example/store.ipynb @PaulJonasJost
 /doc/example/synthetic_data.ipynb @dilpath
 /docker/ @dweindl
+
+# Code
 /pypesto/engine/ @PaulJonasJost
 /pypesto/engine/mpi_pool.py @PaulJonasJost
 /pypesto/ensemble/ @dilpath @PaulJonasJost
 /pypesto/hierarchical/ @dweindl @Doresic
-/pypesto/hierarchical/optimal_scaling_approach/ @Doresic
-/pypesto/hierarchical/spline_approximation/ @Doresic
+/pypesto/hierarchical/ordinal/ @Doresic
+/pypesto/hierarchical/relative/ @dweindl @Doresic
+/pypesto/hierarchical/semiquantitative/ @Doresic
 /pypesto/history/ @PaulJonasJost
 /pypesto/objective/ @PaulJonasJost
 /pypesto/objective/amici/ @dweindl @FFroehlich
@@ -38,6 +42,8 @@
 /pypesto/startpoint/ @PaulJonasJost
 /pypesto/store/ @PaulJonasJost
 /pypesto/visualize/ @stephanmg @m-philipps
+
+# Tests
 /test/base/ @PaulJonasJost @m-philipps
 /test/doc/ @PaulJonasJost
 /test/hierarchical/ @dweindl @doresic

.github/workflows/ci.yml

@@ -5,6 +5,7 @@ on:
   push:
     branches:
     - main
+    - develop
   pull_request:
   workflow_dispatch:
   schedule:
@@ -14,6 +15,8 @@ on:
 env:
   # use all available cores for compiling amici models
   AMICI_PARALLEL_COMPILE: ""
+  # non-interactive backend for matplotlib
+  MPLBACKEND: "agg"
 
 # jobs
 jobs:
@@ -36,7 +39,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
-          ~/.cache
+          ~/.cache/pip
           .tox/
         key: ${{ runner.os }}-${{ matrix.python-version }}-ci-${{ github.job }}
 
@@ -75,9 +78,8 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
-          ~/.cache
+          ~/.cache/pip
           .tox/
-          ~/Library/Caches/Homebrew
         key: ${{ runner.os }}-${{ matrix.python-version }}-ci
 
     - name: Install dependencies
@@ -145,7 +147,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
-          ~/.cache
+          ~/.cache/pip
           .tox/
         key: ${{ runner.os }}-${{ matrix.python-version }}-ci-${{ github.job }}
 
@@ -189,7 +191,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
-          ~/.cache
+          ~/.cache/pip
           .tox/
         key: ${{ runner.os }}-${{ matrix.python-version }}-ci-${{ github.job }}
 
@@ -205,7 +207,9 @@ jobs:
       run: .github/workflows/install_deps.sh
 
     - name: Install PEtabJL dependencies
-      run: julia -e 'using Pkg; Pkg.add("PEtab"); Pkg.add("OrdinaryDiffEq"); Pkg.add("Sundials")'
+      run: >
+        julia -e 'using Pkg; Pkg.add(Pkg.PackageSpec(;name="PEtab", version="2.5.0"));
+        Pkg.add("OrdinaryDiffEq"); Pkg.add("Sundials")'
 
     - name: Run tests
       timeout-minutes: 25
@@ -237,7 +241,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
-          ~/.cache
+          ~/.cache/pip
           .tox/
         key: ${{ runner.os }}-${{ matrix.python-version }}-ci-${{ github.job }}
 
@@ -273,7 +277,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
-          ~/.cache
+          ~/.cache/pip
           .tox/
         key: ${{ runner.os }}-${{ matrix.python-version }}-ci-${{ github.job }}
 
@@ -309,7 +313,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
-          ~/.cache
+          ~/.cache/pip
           .tox/
         key: ${{ runner.os }}-${{ matrix.python-version }}-ci-${{ github.job }}
 
@@ -345,7 +349,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
-          ~/.cache
+          ~/.cache/pip
           .tox/
         key: ${{ runner.os }}-${{ matrix.python-version }}-ci-${{ github.job }}
 
@@ -377,7 +381,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
-          ~/.cache
+          ~/.cache/pip
           .tox/
         key: ${{ runner.os }}-${{ matrix.python-version }}-ci-${{ github.job }}
 
@@ -388,7 +392,7 @@ jobs:
       run: pip install cffconvert && cffconvert --validate
 
     - name: Build doc
-      timeout-minutes: 10
+      timeout-minutes: 30
       run: tox -e doc
 
   notebooks1:
@@ -410,7 +414,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
-          ~/.cache
+          ~/.cache/pip
           .tox/
         key: ${{ runner.os }}-${{ matrix.python-version }}-ci-${{ github.job }}
 
@@ -440,7 +444,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
-          ~/.cache
+          ~/.cache/pip
           .tox/
         key: ${{ runner.os }}-${{ matrix.python-version }}-ci-${{ github.job }}
 

.github/workflows/clear-cache.yml

@@ -0,0 +1,31 @@
+# Save cache space by deleting cache entries from a PR after it was merged.
+# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
+name: Delete after PR merge
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cleanup
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          echo "Fetching list of cache key"
+          cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH -L 100 | cut -f 1 )
+
+          ## Setting this to not fail the workflow while deleting cache keys.
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $cacheKeysForPR
+          do
+              gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          BRANCH: refs/pull/${{ github.event.pull_request.number }}/merge

.github/workflows/install_deps.sh

@@ -10,7 +10,7 @@ pip install wheel setuptools
 pip install tox
 
 # Update package lists
-if [ "$(uname)" == "Darwin" ]; then
+if [ "$(uname)" = "Darwin" ]; then
   # MacOS
   :
 else
@@ -28,7 +28,7 @@ for par in "$@"; do
 
     amici)
       # for amici
-      if [ "$(uname)" == "Darwin" ]; then
+      if [ "$(uname)" = "Darwin" ]; then
         brew install swig hdf5 libomp
       else
         sudo apt-get install \

.readthedocs.yml

@@ -19,9 +19,11 @@ python:
         - doc
 
 build:
-  os: "ubuntu-20.04"
+  os: "ubuntu-22.04"
   apt_packages:
+    - clang
     - libatlas-base-dev
+    - libhdf5-serial-dev
     - swig
   tools:
-    python: "3.10"
+    python: "3.11"

CHANGELOG.rst

@@ -6,6 +6,50 @@ Release notes
 ..........
 
 
+0.4.2 (2024-01-30)
+-------------------
+
+* General
+    * Stabilize tests (#1240, #1254, #1300, #1302, #1303)
+    * Update type annotations and documentations (#1239, #1248, #1255, #1258, #1251, #1268, #1275)
+    * GHA/Codeowner changes (#1260, #1261, #1259, #1262, #1285)
+    * Update utility functions (#1243)
+    * Refactor progress bars (#1272)
+    * Clear Notebook output(#1246, #1277, #1274, #1271, #1276, #1278)
+* Optimize
+    * (Sac)ESSOptimizer: History of best objective values (#1212)
+    * Fix missing fixed parameters in scatter search results (#1265)
+    * Fix TypeError in pypesto.result.optimize.OptimizerResult.summary if x0 is None (#1266)
+    * ESSOptimizer: Include results for local searches in OptimizeResult (#1270)
+* **New Feature**: Spline Approximation (#1222)
+* Select
+    * Allow for hierarchical problems (#1241)
+    * custom minimize method (#1264)
+    * Set estimated parameters in petab_select.Models (#1287)
+* Hierarchical
+    * Log space startpoint sampling (#1242)
+    * Support for box constraints on offset and scaling parameters (#1238)
+    * restructuring and add relative to InnerCalculatorCollector (#1245)
+    * Semiquantitative: Robust regularization calculation (#1297)
+* History
+    * Support pathlib.Path for result/history files (#1247)
+    * Extended Amici history (#1263)
+* Visualize
+    * Fix time trajectories for hierarchical problems (#1213)
+    * Fix hierarchical parameter plotting for all optimizers (#1244)
+    * Sacess history plot (#1250)
+* Objective
+    * Fix PEtab.jl version to before 2.5.0 (temporarily) (#1256)
+* PEtab
+    * Enable Importer passing verbose to create_model (#1269)
+    * PetabImporter: version-specific amici model directories (#1283)
+* Problem
+    * Problem: add inner problem names, bounds and hierarchical flag (#1282)
+    * Use warnings.warn instead of logging.warn when loading Problem from HDF5 without an Objective (#1253)
+* Ensemble
+    * EnsemblePrediction: remove "no predictor" warning (#1293)
+
+
 0.4.1 (2023-12-05)
 -------------------
 

doc/api.rst

@@ -7,8 +7,9 @@ API reference
    pypesto.engine
    pypesto.ensemble
    pypesto.hierarchical
-   pypesto.hierarchical.optimal_scaling
-   pypesto.hierarchical.spline_approximation
+   pypesto.hierarchical.ordinal
+   pypesto.hierarchical.relative
+   pypesto.hierarchical.semiquantitative
    pypesto.history
    pypesto.logging
    pypesto.objective

doc/conf.py

@@ -22,6 +22,20 @@
 
 sys.path.insert(0, os.path.abspath('../'))
 
+# Silence:
+# Debugger warning: It seems that frozen modules are being used, which may
+# make the debugger miss breakpoints. Please pass -Xfrozen_modules=off
+# to python to disable frozen modules.
+# Note: Debugging will proceed. Set PYDEVD_DISABLE_FILE_VALIDATION=1 to disable this validation.
+os.environ["PYDEVD_DISABLE_FILE_VALIDATION"] = "1"
+
+# TQDM and nbsphinx do not play well together. Therefore, disable TQDM
+# for the documentation build.
+# (`Content block expected for the "raw" directive; none found.`)
+os.environ["TQDM_DISABLE"] = "1"
+
+# Enable parallel compilation of AMICI models
+os.environ["AMICI_PARALLEL_COMPILE"] = ""
 
 # -- General configuration ------------------------------------------------
 
@@ -70,24 +84,25 @@
     'show-inheritance': True,
     'autodoc_inherit_docstrings': True,
 }
-autodoc_mock_imports = ["amici"]
 autodoc_class_signature = "separated"
 
 # napoleon options
 napoleon_use_rtype = False
 
 # links for intersphinx
 intersphinx_mapping = {
-    'python': ('https://docs.python.org/3', None),
+    'amici': ('https://amici.readthedocs.io/en/latest/', None),
+    'fides': ('https://fides-optimizer.readthedocs.io/en/latest/', None),
+    'matplotlib': ('https://matplotlib.org/stable/', None),
     'numpy': ('https://numpy.org/devdocs/', None),
-    'scipy': ('https://docs.scipy.org/doc/scipy/', None),
     'pandas': ('https://pandas.pydata.org/docs/', None),
     'petab': (
         'https://petab.readthedocs.io/projects/libpetab-python/en/latest/',
         None,
     ),
-    'amici': ('https://amici.readthedocs.io/en/latest/', None),
-    'fides': ('https://fides-optimizer.readthedocs.io/en/latest/', None),
+    'petab_select': ('https://petab-select.readthedocs.io/en/develop/', None),
+    'python': ('https://docs.python.org/3', None),
+    'scipy': ('https://docs.scipy.org/doc/scipy/', None),
 }
 
 

doc/example.rst

@@ -51,10 +51,10 @@ Algorithms and features
    example/store.ipynb
    example/model_selection.ipynb
    example/julia.ipynb
-   example/hierarchical.ipynb
-   example/ordinal.ipynb
-   example/censored.ipynb
-   example/nonlinear_monotone.ipynb
+   example/relative_data.ipynb
+   example/ordinal_data.ipynb
+   example/censored_data.ipynb
+   example/semiquantitative_data.ipynb
 
 Application examples
 --------------------