tests+pypi.yml

name: tests+pypi

defaults:
  run:
    shell: bash

# arbitrarily selected runs
env:
  animation_run: '3.10/ubuntu-latest/0/openmpi'
  codecov_run: '3.10/ubuntu-latest/1/openmpi'

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
  schedule:
    - cron: '0 13 * * 4'
  release:
    types: [published]

jobs:
  precommit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v1
        with:
          python-version: '3.10'
      - name: Linting
        run: |
          pip install pre-commit
          pre-commit clean
          pre-commit autoupdate
          pre-commit run --all-files
  pdoc:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          persist-credentials: false
      - uses: actions/setup-python@v2
        with:
          python-version: '3.10'
      - uses: mpi4py/setup-mpi@v1
      - name: Build
        run: |
          pip install pdoc
          pip install -e .
          PDOC_ALLOW_EXEC=1 python -We -m pdoc -o html PyMPDATA_MPI
      - name: Deploy
        if: github.ref == 'refs/heads/main'
        uses: JamesIves/github-pages-deploy-action@4.1.1
        with:
          BRANCH: pdoc
          FOLDER: html
          CLEAN: true

  zenodo_json:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: notiz-dev/github-action-json-property@release
        with: 
          path: '.zenodo.json'
          prop_path: 'creators'

  pylint:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
      - uses: mpi4py/setup-mpi@v1
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.10'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -e .[tests]
          pip install pylint pytest matplotlib
      - name: Analysing the code with pylint
        run: |
          pylint --unsafe-load-any-extension=y --disable=fixme $(git ls-files '*.py')

  tests_setup:
    strategy:
      matrix:
        platform: [ubuntu-latest, macos-12]
        mpi: [ 'mpich', 'openmpi', 'intelmpi']
        python-version: ["3.10", "3.12"]
        exclude:
          # as of time of writing, mpi4py/setup-mpi does not support it
          - platform: macos-12
            mpi: intelmpi
	  - platform: macos-12
	    python-version: "3.12"
          # issues with: *** The MPI_Comm_rank() function was called before MPI_INIT was invoked.
          - platform: ubuntu-latest
            mpi: intelmpi

          # https://github.com/Homebrew/homebrew-core/issues/26974
          - platform: macos-12
            mpi: mpich

    runs-on: ${{ matrix.platform }}
    steps:
        - uses: actions/checkout@v2
        - uses: actions/setup-python@v1
          with:
            python-version: ${{ matrix.python-version }}
        - run: |
            echo pip_user_site=$(python -c "import sysconfig; print(sysconfig.get_path('purelib'))") >> $GITHUB_ENV
            echo toml_ci_md5=$(cat pyproject.toml .github/workflows/tests+pypi.yml \
            | python -c "import hashlib;print(hashlib.md5(open(0,'rb').read()).hexdigest())") >> $GITHUB_ENV
        - id: cache
          uses: actions/cache@v4
          with:
            path: ${{ env.pip_user_site }}
            key: ${{ matrix.platform }}-${{ matrix.mpi }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }}

        - if: steps.cache.outputs.cache-hit != 'true'
          uses: mpi4py/setup-mpi@v1
          with:
            mpi: ${{ matrix.mpi }}
        - if: steps.cache.outputs.cache-hit != 'true' && matrix.mpi == 'mpich'
          run: echo _ch="ch" >> $GITHUB_ENV
        - if: steps.cache.outputs.cache-hit != 'true' &&  startsWith(matrix.platform, 'ubuntu-')
          run: |
            sudo apt-get update && sudo apt-get install -y libhdf5-mpi$_ch-dev pkg-config
            lscpu
        - if: steps.cache.outputs.cache-hit != 'true' && startsWith(matrix.platform, 'ubuntu-') && matrix.mpi == 'mpich'
          run: |
            echo HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich >> $GITHUB_ENV
            echo HDF5_INCLUDEDIR=/usr/include/hdf5/mpich >> $GITHUB_ENV
        - if: steps.cache.outputs.cache-hit != 'true' && startsWith(matrix.platform, 'macos-')
          run: | 
            brew install hdf5-mpi && echo HDF5_DIR=/opt/homebrew >> $GITHUB_ENV
            sysctl -a | grep cpu | grep hw
        - if: steps.cache.outputs.cache-hit != 'true'
          run: |
            HDF5_MPI="ON" CC=mpicc pip install --no-binary=h5py h5py==3.12.1
            pip install -e .[tests]
        - run: pip show numpy
        - id: cache-save
          if: steps.cache.outputs.cache-hit != 'true'
          uses: actions/cache/save@v4
          with:
            path: ${{ env.pip_user_site }}
            key: ${{ matrix.platform }}-${{ matrix.mpi }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }}

  tests:
    needs: [zenodo_json, pylint, pdoc, precommit, tests_setup]
    strategy:
      matrix:
        platform: [ubuntu-latest, macos-12]
        mpi: [ 'mpich', 'openmpi', 'intelmpi']
        python-version: ["3.10", "3.12"]
        disable-jit: [1, 0]
        mpi-np: [1, 2, 3]
        exclude:
          # as of time of writing, mpi4py/setup-mpi does not support it
          - platform: macos-12
            mpi: intelmpi
	  - platform: macos-12
	    python-version: "3.12"

          # issues with: *** The MPI_Comm_rank() function was called before MPI_INIT was invoked.
          - platform: ubuntu-latest
            mpi: intelmpi

          # https://github.com/Homebrew/homebrew-core/issues/26974
          - platform: macos-12
            mpi: mpich

      fail-fast: false

    runs-on: ${{ matrix.platform }}
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - run: |
          echo pip_user_site=$(python -c "import sysconfig; print(sysconfig.get_path('purelib'))") >> $GITHUB_ENV
          echo toml_ci_md5=$(cat pyproject.toml .github/workflows/tests+pypi.yml \
          | python -c "import hashlib;print(hashlib.md5(open(0,'rb').read()).hexdigest())") >> $GITHUB_ENV
      - uses: mpi4py/setup-mpi@v1
        with:
          mpi: ${{ matrix.mpi }}
      - id: cache
        uses: actions/cache/restore@v4
        with:
          path: ${{ env.pip_user_site }}
          key: ${{ matrix.platform }}-${{ matrix.mpi }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }}
      - if: matrix.mpi == 'mpich'
        run: echo _ch="ch" >> $GITHUB_ENV
      - if: startsWith(matrix.platform, 'ubuntu-')
        run: sudo apt-get update && sudo apt-get install -y libhdf5-mpi$_ch-dev pkg-config
      - if: startsWith(matrix.platform, 'macos-')
        run: brew install hdf5-mpi && echo HDF5_DIR=/opt/homebrew >> $GITHUB_ENV
      - run: python -We -c "import PyMPDATA_MPI"
      - if: matrix.mpi == 'openmpi'
        run: echo _mpiexec_args="--oversubscribe" >> $GITHUB_ENV
      - name: "mpiexec pytest"
        env:
          NUMBA_DISABLE_JIT: ${{ matrix.disable-jit }} 
        run: |
          if [ "$animation_run" == "${{ matrix.python-version }}/${{ matrix.platform }}/${{ matrix.disable-jit }}/${{ matrix.mpi }}" ]; then
            export CI_PLOTS_PATH=plots/$animation_run
            mkdir -p $CI_PLOTS_PATH
          fi
          if [ "${{ matrix.mpi-np }}" == "1" ] && [ "$codecov_run" == "${{ matrix.python-version }}/${{ matrix.platform }}/${{ matrix.disable-jit }}/${{ matrix.mpi }}" ]; then
            echo "CODECOV_RUN=1" >> $GITHUB_ENV
            export COV_ARGS="--cov=PyMPDATA-MPI --cov=examples --cov=scenarios --cov-report=xml"
            pip install pytest-cov
          fi
          NUMBA_NUM_THREADS=3 mpiexec $_mpiexec_args -n ${{ matrix.mpi-np }} python -m pytest -p no:cacheprovider $COV_ARGS --timeout=600 --timeout_method=thread -s -vv -We tests/local;
      - uses: actions/upload-artifact@v4
        with:
          name: plots-${{matrix.mpi-np}}
          path: plots
      - if: env.CODECOV_RUN == '1'
        uses: codecov/codecov-action@v4
        with: 
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: true
          verbose: true

  package:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0  # https://github.com/pypa/setuptools_scm/issues/480
      - uses: actions/setup-python@v2
        with:
          python-version: "3.10"

      - run: pip install twine build

      - run: |
          unset CI
          python -m build 2>&1 | tee build.log
          exit `fgrep -i warning build.log | wc -l`

      - run: twine check --strict dist/*

      - uses: actions/upload-artifact@v4
        with:
          name: dist
          path: dist

  dist:
    runs-on: ubuntu-latest
    needs: [package,tests]
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: dist
          path: dist

      - if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        uses: pypa/gh-action-pypi-publish@unstable/v1
        with:
          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
          repository_url: https://test.pypi.org/legacy/

      - if: startsWith(github.ref, 'refs/tags')
        uses: pypa/gh-action-pypi-publish@unstable/v1
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}

  devops_tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: recursive
          fetch-depth: 0  # https://github.com/pypa/setuptools_scm/issues/480
      - run: pip install -r tests/devops_tests/requirements.txt
      - env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: pytest --durations=10 -v -s -We -p no:unraisableexception tests/devops_tests

  tip-release-uploads:
    runs-on: ubuntu-latest
    needs: [tests, devops_tests]
    steps:
      - uses: actions/download-artifact@v4
        with:
          pattern: plots-*
          merge-multiple: true
          path: plots
      - run: |
          sudo apt install -y librsvg2-bin
          for scenario in ./plots/${{ env.animation_run }}/*; do
              for test_plots in $scenario/*; do
                  if [ ! $(find "$test_plots" -maxdepth 0 -empty | wc -l) -eq 1 ]; then
                      for file in $test_plots/*.svg; do
                          rsvg-convert $file -o tmp.png
                          convert tmp.png -flatten -trim +repage -bordercolor white -border 10 $file.png
                      done;
                      rm tmp.png
                      convert $test_plots/*.png plots/`basename $test_plots`-`basename $scenario`-anim.gif 
                  fi
              done;
          done;

      - uses: actions/upload-artifact@v4
        with:
          name: anims
          path: plots/*-anim.gif

      - if: github.ref == 'refs/heads/main'
        uses: eine/tip@master
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          files: plots/*-anim.gif
          tag: 'latest-generated-plots'