diff --git a/.github/workflows/check-valid-env.yml b/.github/workflows/check-valid-env.yml index b617bee..738450c 100644 --- a/.github/workflows/check-valid-env.yml +++ b/.github/workflows/check-valid-env.yml @@ -1,4 +1,4 @@ -name: env.yml +name: Check valid env.yml on: pull_request: paths: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..18616f6 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,123 @@ +name: Deploy +on: + push: + branches: + - main + paths: + - 'env.yml' + pull_request: #TODO: remove this line after testing + workflow_dispatch: #TODO: remove this line after testing + +jobs: + setup-deployment-env: + name: Setup Deployment Environment + runs-on: ubuntu-latest + outputs: + deployment-environments: ${{ steps.get-deployment-environment.outputs.deployment-environments }} + steps: + - name: Checkout config + uses: actions/checkout@v4 + + - name: Get environments + id: get-deployment-environment + run: echo "deployment-environments=$(jq --compact-output '.environments' ./config/deployment-environment.json)" >> $GITHUB_OUTPUT + + pack: + name: Pack access-ram-env + runs-on: ubuntu-latest + outputs: + name: ${{ steps.access-ram.outputs.name }} + steps: + - uses: actions/checkout@v4 + + - name: Get Payu Version + id: access-ram + run: echo "name=$(yq '.name' < env.yml)" >> $GITHUB_OUTPUT + + - name: Setup Micromamba + uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 #v1.9.0 + with: + micromamba-version: '1.5.8-0' + environment-file: env.yml + environment-name: ${{ steps.access-ram.outputs.name }} + generate-run-shell: true + + - name: Create Pack and Lockfile + shell: micromamba-shell {0} + run: | + conda pack + # conda-lock lock --file env.yml --platform linux-64 --micromamba --lockfile ${{ steps.access-ram.outputs.name }}.conda-lock.yml + # TODO: Solve issue with conda-lock (https://github.com/ACCESS-NRI/access-ram-condaenv/issues/3) + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.access-ram.outputs.name }} + if-no-files-found: error + path: | + ${{ steps.access-ram.outputs.name }}.tar.gz + # ${{ steps.access-ram.outputs.name }}.conda-lock.yml + # TODO: Solve issue with conda-lock (https://github.com/ACCESS-NRI/access-ram-condaenv/issues/3) + + deploy: + runs-on: ubuntu-latest + needs: + - pack + - setup-deployment-env + strategy: + fail-fast: false + matrix: + deployment-environment: ${{ fromJson(needs.setup-deployment-env.outputs.deployment-environments) }} + environment: ${{ matrix.deployment-environment }} + env: + NAME: ${{ needs.pack.outputs.name }} + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + name: ${{ env.NAME }} + + - name: Set up SSH + uses: access-nri/actions/.github/actions/setup-ssh@main + id: ssh + with: + hosts: | + ${{ secrets.HOST_DATA }} + ${{ secrets.HOST }} + private-key: ${{ secrets.SSH_KEY }} + + - name: Copy to ${{ matrix.deployment-environment }} + run: | + rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ + ${{ env.NAME }}.tar.gz \ + ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.DEPLOYMENT_LOCATION }} + + - name: Deploy to ${{ matrix.deployment-environment }} + env: + ENVIRONMENT_LOCATION: ${{ vars.DEPLOYMENT_LOCATION }} + run: | + ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + mkdir ${{ env.ENVIRONMENT_LOCATION }} + if [ $? -ne 0 ]; then + exit $? + fi + tar -xzf ${{ vars.DEPLOYMENT_LOCATION }}/${{ env.NAME }}.tar.gz -C ${{ env.ENVIRONMENT_LOCATION }} + source ${{ env.ENVIRONMENT_LOCATION }}/bin/activate + conda-unpack + source ${{ env.ENVIRONMENT_LOCATION }}/bin/deactivate + # ln -s ${{ vars.MODULE_LOCATION }}/.common ${{ vars.MODULE_LOCATION }}/${{ env.VERSION }} + EOT + + # Release + # - name: Create Release + # uses: softprops/action-gh-release@v0.1.15 + # with: + # tag_name: ${{ env.VERSION }} + # name: Payu ${{ env.VERSION }} + # generate_release_notes: true + # fail_on_unmatched_files: true + # files: | + # ./${{ env.NAME }}.tar.gz + # # ./${{ env.NAME }}.conda-lock.yml + # # TODO: Solve issue with conda-lock (https://github.com/ACCESS-NRI/access-ram-condaenv/issues/3) \ No newline at end of file diff --git a/README.md b/README.md index 9522c6f..7df6c03 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ New deployment environments must be created as a GitHub Environment and also hav ### Deploying locally -To deploy locally, you can use the assets created during the release. [Releases are found here](https://github.com/ACCESS-NRI/access-ram-condaenv/releases). Specifically: +To deploy locally, you can use the assets created in the release. [Releases are found here](https://github.com/ACCESS-NRI/access-ram-condaenv/releases). Specifically: - To use the compressed environment (which doesn't require conda or python) you can run `tar -xzf access-ram.tar.gz access-ram` and then `./access-ram/bin/activate` to activate the environment. - To use the lockfile, you can run `micromamba create -n environment-name -f access-ram.conda-lock.yml` with an appropriate install of `micromamba`. diff --git a/env-dev.yml b/env-dev.yml index 76fe706..b30cd13 100644 --- a/env-dev.yml +++ b/env-dev.yml @@ -11,6 +11,7 @@ dependencies: - python=3.10 - pytest - pytest-cov + - pre_commit - scitools-iris - xarray - pip: diff --git a/env.yml b/env.yml index 438476d..656b651 100644 --- a/env.yml +++ b/env.yml @@ -4,6 +4,8 @@ channels: - coecms - conda-forge dependencies: + - conda-lock + - conda-pack - numpy=1.23.4 # https://stackoverflow.com/a/75148219/21024780 - pip - python=3.10 diff --git a/modules/.common b/modules/.common index eef2c7f..1ca0724 100644 --- a/modules/.common +++ b/modules/.common @@ -3,17 +3,17 @@ # This is adapted from hh5's modulefiles for activating conda environments # using modules (see /g/data/hh5/public/modules/conda/.common.v2) -module-whatis {Activate access-ram micromamba environment (https://github.com/ACCESS-NRI/access-ram-condaenv)} +module-whatis {Activate micromamba environment to run access-ram python scripts (https://github.com/ACCESS-NRI/access-ram-condaenv)} conflict access-ram # Name of this module's environment set envname [module-info name] # Get the environment variables from activate script -set payuenv [exec /bin/env -i {{MODULE_LOCATION}}/env.sh $envname] +set environment [exec /bin/env -i {{MODULE_LOCATION}}/env.sh $envname] # Convert the environment into module commands -set lines [split $payuenv '\n'] +set lines [split $environment '\n'] foreach line $lines { regexp {^([^=]*)=(.*)} $line -> key value