From 6ba6882ff5b91060898787a99cbaafab77876a6d Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Wed, 14 Aug 2024 14:31:38 +1000 Subject: [PATCH] Updated 'deploy.yml' for testing. --- .github/workflows/deploy.yml | 119 +++++++++++++++++++++++++++++++++++ README.md | 2 +- env.yml | 6 +- modules/.common | 6 +- 4 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..5684628 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,119 @@ +name: Deploy +on: + push: + branches: + - main + paths: + - 'env.yml' + pull_request: #TODO: remove this line after testing + workflow_dispatch: #TODO: remove this line after testing + +jobs: + setup-deployment-env: + name: Setup Deployment Environment + runs-on: ubuntu-latest + outputs: + deployment-environments: ${{ steps.get-deployment-environment.outputs.deployment-environments }} + steps: + - name: Checkout config + uses: actions/checkout@v4 + + - name: Get environments + id: get-deployment-environment + run: echo "deployment-environments=$(jq --compact-output '.environments' ./config/deployment-environment.json)" >> $GITHUB_OUTPUT + + pack: + name: Pack access-ram-env + runs-on: ubuntu-latest + outputs: + name: ${{ steps.access-ram.outputs.name }} + steps: + - uses: actions/checkout@v4 + + - name: Get Payu Version + id: access-ram + run: echo "name=$(yq '.name' < env.yml)" >> $GITHUB_OUTPUT + + - name: Setup Micromamba + uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 #v1.9.0 + with: + micromamba-version: '1.5.8-0' + environment-file: env.yml + environment-name: ${{ steps.access-ram.outputs.name }} + generate-run-shell: true + + - name: Create Pack and Lockfile + shell: micromamba-shell {0} + run: | + conda pack + conda-lock lock --file env.yml --platform linux-64 --micromamba --lockfile ${{ steps.access-ram.outputs.name }}.conda-lock.yml + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.access-ram.outputs.name }} + if-no-files-found: error + path: | + ${{ steps.access-ram.outputs.name }}.tar.gz + ${{ steps.access-ram.outputs.name }}.conda-lock.yml + + deploy: + runs-on: ubuntu-latest + needs: + - pack + - setup-deployment-env + strategy: + fail-fast: false + matrix: + deployment-environment: ${{ fromJson(needs.setup-deployment-env.outputs.deployment-environments) }} + environment: ${{ matrix.deployment-environment }} + env: + NAME: ${{ needs.pack.outputs.name }} + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + name: ${{ env.NAME }} + + - uses: access-nri/actions/.github/actions/setup-ssh@main + id: ssh + with: + hosts: | + ${{ secrets.HOST_DATA }} + ${{ secrets.HOST }} + private-key: ${{ secrets.SSH_KEY }} + + - name: Copy to ${{ matrix.deployment-environment }} + run: | + rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ + ${{ env.NAME }}.tar.gz \ + ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.PACK_LOCATION }} + + - name: Deploy to ${{ matrix.deployment-environment }} + env: + ENVIRONMENT_LOCATION: ${{ vars.DEPLOYMENT_LOCATION }} + run: | + ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + mkdir ${{ env.ENVIRONMENT_LOCATION }} + if [ $? -ne 0 ]; then + exit $? + fi + tar -xzf ${{ vars.PACK_LOCATION }}/${{ env.NAME }}.tar.gz -C ${{ env.ENVIRONMENT_LOCATION }} + source ${{ env.ENVIRONMENT_LOCATION }}/bin/activate + conda-unpack + source ${{ env.ENVIRONMENT_LOCATION }}/bin/deactivate + # ln -s ${{ vars.MODULE_LOCATION }}/.common ${{ vars.MODULE_LOCATION }}/${{ env.VERSION }} + EOT + + # Release + # - name: Create Release + # uses: softprops/action-gh-release@v0.1.15 + # with: + # tag_name: ${{ env.VERSION }} + # name: Payu ${{ env.VERSION }} + # generate_release_notes: true + # fail_on_unmatched_files: true + # files: | + # ./${{ env.NAME }}.tar.gz + # ./${{ env.NAME }}.conda-lock.yml \ No newline at end of file diff --git a/README.md b/README.md index 9522c6f..7df6c03 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ New deployment environments must be created as a GitHub Environment and also hav ### Deploying locally -To deploy locally, you can use the assets created during the release. [Releases are found here](https://github.com/ACCESS-NRI/access-ram-condaenv/releases). Specifically: +To deploy locally, you can use the assets created in the release. [Releases are found here](https://github.com/ACCESS-NRI/access-ram-condaenv/releases). Specifically: - To use the compressed environment (which doesn't require conda or python) you can run `tar -xzf access-ram.tar.gz access-ram` and then `./access-ram/bin/activate` to activate the environment. - To use the lockfile, you can run `micromamba create -n environment-name -f access-ram.conda-lock.yml` with an appropriate install of `micromamba`. diff --git a/env.yml b/env.yml index 438476d..21fd7b5 100644 --- a/env.yml +++ b/env.yml @@ -4,10 +4,12 @@ channels: - coecms - conda-forge dependencies: + - conda-lock + - conda-pack - numpy=1.23.4 # https://stackoverflow.com/a/75148219/21024780 - pip - python=3.10 - scitools-iris - xarray - - pip: - - mule @ git+https://github.com/metomi/mule@cce4b99c7046217b1ec1192118a786636e0d8e54#subdirectory=mule + # - pip: + # - mule @ git+https://github.com/metomi/mule@cce4b99c7046217b1ec1192118a786636e0d8e54#subdirectory=mule diff --git a/modules/.common b/modules/.common index eef2c7f..1ca0724 100644 --- a/modules/.common +++ b/modules/.common @@ -3,17 +3,17 @@ # This is adapted from hh5's modulefiles for activating conda environments # using modules (see /g/data/hh5/public/modules/conda/.common.v2) -module-whatis {Activate access-ram micromamba environment (https://github.com/ACCESS-NRI/access-ram-condaenv)} +module-whatis {Activate micromamba environment to run access-ram python scripts (https://github.com/ACCESS-NRI/access-ram-condaenv)} conflict access-ram # Name of this module's environment set envname [module-info name] # Get the environment variables from activate script -set payuenv [exec /bin/env -i {{MODULE_LOCATION}}/env.sh $envname] +set environment [exec /bin/env -i {{MODULE_LOCATION}}/env.sh $envname] # Convert the environment into module commands -set lines [split $payuenv '\n'] +set lines [split $environment '\n'] foreach line $lines { regexp {^([^=]*)=(.*)} $line -> key value