Merge pull request #2 from ACCESS-NRI/update_deploy

Updated 'deploy.yml' for testing.
ACCESS-NRI · Aug 20, 2024 · 607b33e · 607b33e
2 parents 0e5a978 + de5954f
commit 607b33e
Show file tree

Hide file tree

Showing 9 changed files with 241 additions and 21 deletions.
diff --git a/.github/workflows/check-valid-env.yml b/.github/workflows/check-valid-env.yml
@@ -1,11 +1,11 @@
-name: env.yml
+name: Check valid env.yml
 on:
   pull_request:
     paths:
       - 'env.yml'
 jobs:
-  check:
-    name: Check
+  check-env:
+    name: Check env.yml
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -16,4 +16,31 @@ jobs:
           micromamba-version: '1.5.8-0'
           environment-file: env.yml
           environment-name: test-access-ram-environment
-          generate-run-shell: false
+          generate-run-shell: false
+
+  # TODO: Remove this once we have a better versioning solution than a VERSION file
+  check-version:
+    name: Check VERSION
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36  # v3.0.2
+        id: filter
+        with:
+          filters: |
+            version:
+              - 'VERSION'
+
+      - name: Comment if VERSION not modified
+        if: steps.filter.outputs.version == 'false'
+        uses: access-nri/actions/.github/actions/pr-comment@main
+        with:
+          comment: |
+            :x: `VERSION` file must be updated before this PR is merged to deploy :x:
+
+      - name: Error if VERSION not modified
+        if: steps.filter.outputs.version == 'false'
+        run: |
+          echo "::error::Version must be modified in this PR"
+          exit 1
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -0,0 +1,134 @@
+name: Deploy
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'env.yml'
+jobs:
+  setup-deployment-env:
+    name: Setup Deployment Environment
+    runs-on: ubuntu-latest
+    outputs:
+      deployment-environments: ${{ steps.get-deployment-environment.outputs.deployment-environments }}
+    steps:
+      - name: Checkout config
+        uses: actions/checkout@v4
+
+      - name: Get environments
+        id: get-deployment-environment
+        run: echo "deployment-environments=$(jq --compact-output '.environments' ./config/deployment-environment.json)" >> $GITHUB_OUTPUT
+
+  # TODO: Change from VERSION file to something more robust in future
+  version:
+    name: Set release version
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set version
+        id: version
+        # cat the version file and remove newlines, just in case
+        run: echo "version=$(cat VERSION | tr -d '\n')" >> $GITHUB_OUTPUT
+
+  pack:
+    name: Pack access-ram-env
+    runs-on: ubuntu-latest
+    needs: version
+    env:
+      VERSION: ${{ needs.version.outputs.version }}
+    outputs:
+      name: ${{ steps.access-ram.outputs.name }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get name
+        id: access-ram
+        run: echo "name=$(yq '.name' < env.yml)" >> $GITHUB_OUTPUT
+
+      - name: Setup Micromamba
+        uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 #v1.9.0
+        with:
+          micromamba-version: '1.5.8-0'
+          environment-file: env.yml
+          environment-name: ${{ steps.access-ram.outputs.name }}-${{env.VERSION}}
+          generate-run-shell: true
+
+      - name: Create Pack and Lockfile
+        shell: micromamba-shell {0}
+        run: |
+          conda pack -o ${{ steps.access-ram.outputs.name }}-${{env.VERSION}}.tar.gz
+          conda-lock lock --file env.yml --platform linux-64 --micromamba --lockfile ${{ steps.access-ram.outputs.name }}-${{env.VERSION}}.conda-lock.yml
+
+      - name: Upload Artifact
+        env:
+          PACKAGE_NAME: ${{ steps.access-ram.outputs.name }}-${{ env.VERSION }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.PACKAGE_NAME }}
+          if-no-files-found: error
+          path: |
+            ${{ env.PACKAGE_NAME }}.tar.gz
+            ${{ env.PACKAGE_NAME }}.conda-lock.yml
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs:
+      - version
+      - pack
+      - setup-deployment-env
+    strategy:
+      fail-fast: false
+      matrix:
+        deployment-environment: ${{ fromJson(needs.setup-deployment-env.outputs.deployment-environments) }}
+    environment: ${{ matrix.deployment-environment }}
+    env:
+      VERSION: ${{ needs.version.outputs.version }}
+      PACKAGE_NAME: ${{ needs.pack.outputs.name }}-${{ needs.version.outputs.version }}
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ env.PACKAGE_NAME }}
+
+      - name: Set up SSH
+        uses: access-nri/actions/.github/actions/setup-ssh@main
+        id: ssh
+        with:
+          hosts: |
+            ${{ secrets.HOST_DATA }}
+            ${{ secrets.HOST }}
+          private-key: ${{ secrets.SSH_KEY }}
+
+      - name: Copy to ${{ matrix.deployment-environment }}
+        run: |
+          rsync -v -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
+            ${{ env.PACKAGE_NAME }}.tar.gz \
+            ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.PACK_DIR }}
+
+      - name: Deploy to ${{ matrix.deployment-environment }}
+        env:
+          DEPLOYMENT_DIR: ${{ vars.ENVIRONMENT_DIR }}/${{ env.VERSION }}
+        run: |
+          ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
+          mkdir -p ${{ env.DEPLOYMENT_DIR }}
+          tar -xzf ${{ vars.PACK_DIR }}/${{ env.PACKAGE_NAME }}.tar.gz -C ${{ env.DEPLOYMENT_DIR }}
+          source ${{ env.DEPLOYMENT_DIR }}/bin/activate
+          conda-unpack
+          source ${{ env.DEPLOYMENT_DIR }}/bin/deactivate
+          ln -sf ${{ vars.MODULE_DIR }}/.common ${{ vars.MODULE_DIR }}/${{ env.VERSION }}
+          EOT
+
+      - name: Create Release
+        uses: softprops/action-gh-release@c062e08bd532815e2082a85e87e3ef29c3e6d191 #v2.0.8
+        with:
+          tag_name: ${{ env.VERSION }}
+          name: ACCESS-RAM conda environment ${{ env.VERSION }}
+          generate_release_notes: true
+          fail_on_unmatched_files: true
+          files: |
+            ./${{ env.PACKAGE_NAME }}.tar.gz
+            ./${{ env.PACKAGE_NAME }}.conda-lock.yml
diff --git a/.github/workflows/update-module.yml b/.github/workflows/update-module.yml
@@ -0,0 +1,59 @@
+name: Update common modulefiles on Gadi
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'modules/**'
+jobs:
+  setup-deployment-env:
+    name: Setup Deployment Environment
+    runs-on: ubuntu-latest
+    outputs:
+      deployment-environments: ${{ steps.get-deployment-environment.outputs.deployment-environments }}
+    steps:
+      - name: Checkout config
+        uses: actions/checkout@v4
+
+      - name: Get environments
+        id: get-deployment-environment
+        run: echo "deployment-environments=$(jq --compact-output '.environments' ./config/deployment-environment.json)" >> $GITHUB_OUTPUT
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: setup-deployment-env
+    strategy:
+      fail-fast: false
+      matrix:
+        deployment-environment: ${{ fromJson(needs.setup-deployment-env.outputs.deployment-environments) }}
+    environment: ${{ matrix.deployment-environment }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: access-nri/actions/.github/actions/setup-ssh@main
+        id: ssh
+        with:
+          hosts: |
+            ${{ secrets.HOST_DATA }}
+          private-key: ${{ secrets.SSH_KEY }}
+
+      - name: Generate release modulesfiles
+        run: |
+          mkdir -p release/modules
+          sed 's|{{MODULE_DIR}}|'"${{ vars.MODULE_DIR }}"'|g' modules/.common > release/modules/.common
+          sed 's|{{APPS_DIR}}|'"${{ vars.APPS_DIR }}"'|g' modules/env.sh > release/modules/env.sh
+          chmod +x release/modules/env.sh
+
+      - name: Create deployment directory on ${{ matrix.deployment-environment }}
+        run: |
+          ssh ${{ secrets.USER }}@${{ secrets.HOST_DATA }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
+          mkdir -p ${{ vars.MODULE_DIR }}
+          EOT
+
+      - name: Copy modulefiles to Gadi
+        run: |
+          # Rsync release modulefiles
+          rsync -rvp -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
+            release/modules/ \
+            ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.MODULE_DIR }}
diff --git a/README.md b/README.md
@@ -30,8 +30,8 @@ On all of the deployment targets, the deployed environment can be activated usin
 2. Once you are a member, run the following:
 
    ```bash
-   module use /g/data/<PROJECT_TO_BE_DEFINED>/modules
-   module load access-ram3
+   module use /g/data/vk83/modules
+   module load conda/access-ram3/<VERSION>
    ```
 
 You are set to run ACCESS-RAM3.
@@ -44,7 +44,7 @@ New deployment environments must be created as a GitHub Environment and also hav
 
 ### Deploying locally
 
-To deploy locally, you can use the assets created during the release. [Releases are found here](https://github.com/ACCESS-NRI/access-ram-condaenv/releases). Specifically:
+To deploy locally, you can use the assets created in the release. [Releases are found here](https://github.com/ACCESS-NRI/access-ram-condaenv/releases). Specifically:
 
-- To use the compressed environment (which doesn't require conda or python) you can run `tar -xzf access-ram.tar.gz access-ram` and then `./access-ram/bin/activate` to activate the environment.
+- To use the compressed environment (which doesn't require conda or python) you can run `tar -xzf access-ram-<VERSION>.tar.gz access-ram` and then `./access-ram/bin/activate` to activate the environment.
 - To use the lockfile, you can run `micromamba create -n environment-name -f access-ram.conda-lock.yml` with an appropriate install of `micromamba`.
diff --git a/VERSION b/VERSION
@@ -0,0 +1 @@
+1.0
diff --git a/env-dev.yml b/env-dev.yml
@@ -6,12 +6,11 @@ channels:
 dependencies:
   - hypothesis
   - ipykernel
+  - mule
   - numpy=1.23.4 # https://stackoverflow.com/a/75148219/21024780
-  - pip
   - python=3.10
   - pytest
   - pytest-cov
+  - pre_commit
   - scitools-iris
-  - xarray
-  - pip:
-    - mule @ git+https://github.com/metomi/mule@cce4b99c7046217b1ec1192118a786636e0d8e54#subdirectory=mule
+  - xarray
diff --git a/env.yml b/env.yml
@@ -4,10 +4,10 @@ channels:
   - coecms
   - conda-forge
 dependencies:
+  - conda-lock
+  - conda-pack
+  - mule
   - numpy=1.23.4 # https://stackoverflow.com/a/75148219/21024780
-  - pip
   - python=3.10
   - scitools-iris
   - xarray
-  - pip:
-    - mule @ git+https://github.com/metomi/mule@cce4b99c7046217b1ec1192118a786636e0d8e54#subdirectory=mule
diff --git a/modules/.common b/modules/.common
@@ -3,17 +3,17 @@
 # This is adapted from hh5's modulefiles for activating conda environments
 # using modules (see /g/data/hh5/public/modules/conda/.common.v2)
 
-module-whatis {Activate access-ram micromamba environment (https://github.com/ACCESS-NRI/access-ram-condaenv)}
-conflict access-ram
+module-whatis {Activate micromamba environment to run access-ram python scripts (https://github.com/ACCESS-NRI/access-ram-condaenv)}
+conflict conda
 
-# Name of this module's environment
+# Name of this module's environment (e.g. conda/access-ram-VERSION)
 set envname [module-info name]
 
 # Get the environment variables from activate script
-set payuenv [exec /bin/env -i {{MODULE_LOCATION}}/env.sh $envname]
+set environment [exec /bin/env -i {{MODULE_DIR}}/env.sh $envname]
 
 # Convert the environment into module commands
-set lines [split $payuenv '\n']
+set lines [split $environment '\n']
 foreach line $lines {
     regexp {^([^=]*)=(.*)} $line -> key value
 

diff --git a/modules/env.sh b/modules/env.sh
@@ -2,5 +2,5 @@
 # Prints the environment variables when activating un-packed environment
 export PATH=/usr/bin:/bin
 
-source {{APPS_LOCATION}}/$1/bin/activate
+source {{APPS_DIR}}/$1/bin/activate
 /bin/env