Build Recipes #228
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Recipes | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| schedule: | |
| # Twice per week, on Monday and Thursday | |
| - cron: "0 0 * * 1,4" | |
| permissions: | |
| contents: write | |
| env: | |
| # Increase this value to reset cache if environment.yml has not changed | |
| CONDA_CACHE_NUMBER: 0 | |
| # Increase this value to reset cache if emscripten_version has not changed | |
| EMSDK_CACHE_FOLDER: 'emsdk-cache' | |
| EMSDK_CACHE_NUMBER: 0 | |
| CCACHE_DIR: /tmp/.ccache | |
| CCACHE_CACHE_NUMBER: 0 | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install Emscripten ccache | |
| shell: bash -l {0} | |
| run: | | |
| # FIXME: Installing ccache using `emsdk install ccache-git-emscripten-64bit` doesn't work well in conda env: | |
| # https://stackoverflow.com/questions/71340058/conda-does-not-look-for-libpthread-and-libpthread-nonshared-at-the-right-place-w | |
| git clone https://github.com/juj/ccache -b emscripten --depth 1 | |
| cd ccache | |
| cmake . | |
| make ccache | |
| export PATH=$(pwd):$PATH | |
| cd .. | |
| which ccache | |
| - name: Cache conda | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/conda_pkgs_dir | |
| key: | |
| ${{ runner.os }}-conda-${{ env.CONDA_CACHE_NUMBER }}-${{ | |
| hashFiles('environment.yml') }} | |
| - name: Setup Mambaforge | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| miniforge-variant: Mambaforge | |
| miniforge-version: latest | |
| activate-environment: pyodide-env | |
| use-mamba: true | |
| - name: Get Date | |
| id: get-date | |
| run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Cache Conda env | |
| id: conda-cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: ${{ env.CONDA }}/envs | |
| key: | |
| conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ hashFiles('environment.yml') }}-${{ env.CONDA_CACHE_NUMBER }} | |
| - name: Update environment if cache miss | |
| run: | |
| mamba env update -n pyodide-env -f environment.yml | |
| if: steps.conda-cache.outputs.cache-hit != 'true' | |
| - name: Install latest pyodide-build and build dependencies | |
| shell: bash -l {0} | |
| run : | | |
| which python | |
| python -m pip install git+https://github.com/pyodide/pyodide.git@main#subdirectory=pyodide-build | |
| pyodide xbuildenv install --url http://pyodide-cache.s3-website-us-east-1.amazonaws.com/xbuildenv/dev/xbuildenv.tar.bz2 | |
| - name: Check emscripten version | |
| shell: bash -l {0} | |
| run: | | |
| echo EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) >> $GITHUB_ENV | |
| - name: Cache emsdk | |
| uses: actions/cache@v2 | |
| with: | |
| path: ${{env.EMSDK_CACHE_FOLDER}} | |
| key: ${{env.EMSDK_CACHE_NUMBER}}-${{env.EMSCRIPTEN_VERSION}}-${{ runner.os }} | |
| - uses: mymindstorm/setup-emsdk@v11 | |
| with: | |
| version: ${{ env.EMSCRIPTEN_VERSION }} | |
| actions-cache-folder: ${{env.EMSDK_CACHE_FOLDER}} | |
| - name: Set ccache suffix | |
| shell: bash -l {0} | |
| run: | | |
| # This step makes the cache key in the main branch and PRs different. | |
| # main branch run a full build, while PRs run a partial build | |
| # so we keep the ccache cache separate | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| echo CCACHE_SUFFIX="-pr" >> $GITHUB_ENV | |
| fi | |
| - name: Cache ccache | |
| uses: actions/cache@v2 | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| key: ${{ env.CCACHE_DIR }}-${{ env.CCACHE_CACHE_NUMBER }}-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }}${{ env.CCACHE_SUFFIX }} | |
| restore-keys: | | |
| ${{ env.CCACHE_DIR }}-${{ env.CCACHE_CACHE_NUMBER }}-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }} | |
| - name: Calculate recipes to build (pull_request) | |
| if: github.event_name == 'pull_request' | |
| id: calculate_recipes_pr | |
| shell: bash -l {0} | |
| run: | | |
| export CHANGED_RECIPES=$(python ./tools/calc_diff.py \ | |
| --base origin/${{ github.event.pull_request.base.ref }} \ | |
| --target ${{ github.sha }}) | |
| echo "Changed recipes: $CHANGED_RECIPES" | |
| # If there are no changed recipes, we build only core packages sets | |
| if [ -z "$CHANGED_RECIPES" ]; then | |
| echo "recipes=tag:core" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "recipes=$CHANGED_RECIPES,tag:core" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Build recipes (full) | |
| shell: bash -l {0} | |
| if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[full build]') }} | |
| run: | | |
| ccache -z | |
| export _EMCC_CACHE=1 | |
| export PIP_CONSTRAINT=$(pwd)/tools/constraints.txt | |
| pyodide build-recipes "*" --install --install-dir=./repodata | |
| ccache -s | |
| - name: Build recipes (changed only) | |
| shell: bash -l {0} | |
| if: github.event_name == 'pull_request' && !contains(github.event.head_commit.message, '[full build]') | |
| run: | | |
| ccache -z | |
| export _EMCC_CACHE=1 | |
| export PIP_CONSTRAINT=$(pwd)/tools/constraints.txt | |
| pyodide build-recipes ${{ steps.calculate_recipes_pr.outputs.recipes }} --install --install-dir=./repodata | |
| ccache -s | |
| - name: Store artifacts build | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: repodata | |
| path: ./repodata/ | |
| retention-days: 15 | |
| - name: Release | |
| uses: softprops/action-gh-release@v1 | |
| if: startsWith(github.ref, 'refs/tags/') | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| files: | | |
| ./repodata/*.whl | |
| test: | |
| runs-on: ubuntu-latest | |
| needs: [build] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| test-config: [ | |
| {runner: selenium, runtime: chrome, runtime-version: latest }, | |
| ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Cache conda | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/conda_pkgs_dir | |
| key: | |
| ${{ runner.os }}-conda-${{ env.CONDA_CACHE_NUMBER }}-${{ | |
| hashFiles('environment.yml') }} | |
| - uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| activate-environment: pyodide-env | |
| environment-file: environment.yml | |
| channels: conda-forge | |
| use-only-tar-bz2: true | |
| - name: Download latest Pyodide | |
| shell: bash -l {0} | |
| run: | | |
| wget http://pyodide-cache.s3-website-us-east-1.amazonaws.com/xbuildenv/dev/pyodide-core.tar.bz2 | |
| tar -xvf pyodide-core.tar.bz2 | |
| mv pyodide dist | |
| - uses: pyodide/pyodide-actions/install-browser@v1 | |
| with: | |
| runner: ${{ matrix.test-config.runner }} | |
| browser: ${{ matrix.test-config.runtime }} | |
| browser-version: ${{ matrix.test-config.runtime-version }} | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: repodata | |
| path: ./repodata/ | |
| - name: Copy repodata | |
| run: | | |
| cp ./repodata/* ./dist/ | |
| - name: Run tests | |
| shell: bash -l {0} | |
| run: | | |
| # FIXME: don't skip webworker tests | |
| # FIXKE: don't skip cpp_exception tests | |
| pytest -v \ | |
| --dist-dir=./dist/ \ | |
| --runner=${{ matrix.test-config.runner }} \ | |
| --rt ${{ matrix.test-config.runtime }} \ | |
| -k "not webworker and not cpp_exception and not triangulation and not cartopy" \ | |
| packages |