diff --git a/.github/workflows/check-valid-env.yml b/.github/workflows/check-valid-env.yml index b617bee..0f511b2 100644 --- a/.github/workflows/check-valid-env.yml +++ b/.github/workflows/check-valid-env.yml @@ -1,11 +1,11 @@ -name: env.yml +name: Check valid env.yml on: pull_request: paths: - 'env.yml' jobs: - check: - name: Check + check-env: + name: Check env.yml runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -16,4 +16,31 @@ jobs: micromamba-version: '1.5.8-0' environment-file: env.yml environment-name: test-access-ram-environment - generate-run-shell: false \ No newline at end of file + generate-run-shell: false + + # TODO: Remove this once we have a better versioning solution than a VERSION file + check-version: + name: Check VERSION + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + version: + - 'VERSION' + + - name: Comment if VERSION not modified + if: steps.filter.outputs.version == 'false' + uses: access-nri/actions/.github/actions/pr-comment@main + with: + comment: | + :x: `VERSION` file must be updated before this PR is merged to deploy :x: + + - name: Error if VERSION not modified + if: steps.filter.outputs.version == 'false' + run: | + echo "::error::Version must be modified in this PR" + exit 1 \ No newline at end of file diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..d96743b --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,134 @@ +name: Deploy +on: + push: + branches: + - main + paths: + - 'env.yml' +jobs: + setup-deployment-env: + name: Setup Deployment Environment + runs-on: ubuntu-latest + outputs: + deployment-environments: ${{ steps.get-deployment-environment.outputs.deployment-environments }} + steps: + - name: Checkout config + uses: actions/checkout@v4 + + - name: Get environments + id: get-deployment-environment + run: echo "deployment-environments=$(jq --compact-output '.environments' ./config/deployment-environment.json)" >> $GITHUB_OUTPUT + + # TODO: Change from VERSION file to something more robust in future + version: + name: Set release version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v4 + + - name: Set version + id: version + # cat the version file and remove newlines, just in case + run: echo "version=$(cat VERSION | tr -d '\n')" >> $GITHUB_OUTPUT + + pack: + name: Pack access-ram-env + runs-on: ubuntu-latest + needs: version + env: + VERSION: ${{ needs.version.outputs.version }} + outputs: + name: ${{ steps.access-ram.outputs.name }} + steps: + - uses: actions/checkout@v4 + + - name: Get name + id: access-ram + run: echo "name=$(yq '.name' < env.yml)" >> $GITHUB_OUTPUT + + - name: Setup Micromamba + uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 #v1.9.0 + with: + micromamba-version: '1.5.8-0' + environment-file: env.yml + environment-name: ${{ steps.access-ram.outputs.name }}-${{env.VERSION}} + generate-run-shell: true + + - name: Create Pack and Lockfile + shell: micromamba-shell {0} + run: | + conda pack -o ${{ steps.access-ram.outputs.name }}-${{env.VERSION}}.tar.gz + conda-lock lock --file env.yml --platform linux-64 --micromamba --lockfile ${{ steps.access-ram.outputs.name }}-${{env.VERSION}}.conda-lock.yml + + - name: Upload Artifact + env: + PACKAGE_NAME: ${{ steps.access-ram.outputs.name }}-${{ env.VERSION }} + uses: actions/upload-artifact@v4 + with: + name: ${{ env.PACKAGE_NAME }} + if-no-files-found: error + path: | + ${{ env.PACKAGE_NAME }}.tar.gz + ${{ env.PACKAGE_NAME }}.conda-lock.yml + + deploy: + runs-on: ubuntu-latest + needs: + - version + - pack + - setup-deployment-env + strategy: + fail-fast: false + matrix: + deployment-environment: ${{ fromJson(needs.setup-deployment-env.outputs.deployment-environments) }} + environment: ${{ matrix.deployment-environment }} + env: + VERSION: ${{ needs.version.outputs.version }} + PACKAGE_NAME: ${{ needs.pack.outputs.name }}-${{ needs.version.outputs.version }} + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + name: ${{ env.PACKAGE_NAME }} + + - name: Set up SSH + uses: access-nri/actions/.github/actions/setup-ssh@main + id: ssh + with: + hosts: | + ${{ secrets.HOST_DATA }} + ${{ secrets.HOST }} + private-key: ${{ secrets.SSH_KEY }} + + - name: Copy to ${{ matrix.deployment-environment }} + run: | + rsync -v -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ + ${{ env.PACKAGE_NAME }}.tar.gz \ + ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.PACK_DIR }} + + - name: Deploy to ${{ matrix.deployment-environment }} + env: + DEPLOYMENT_DIR: ${{ vars.ENVIRONMENT_DIR }}/${{ env.VERSION }} + run: | + ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + mkdir -p ${{ env.DEPLOYMENT_DIR }} + tar -xzf ${{ vars.PACK_DIR }}/${{ env.PACKAGE_NAME }}.tar.gz -C ${{ env.DEPLOYMENT_DIR }} + source ${{ env.DEPLOYMENT_DIR }}/bin/activate + conda-unpack + source ${{ env.DEPLOYMENT_DIR }}/bin/deactivate + ln -sf ${{ vars.MODULE_DIR }}/.common ${{ vars.MODULE_DIR }}/${{ env.VERSION }} + EOT + + - name: Create Release + uses: softprops/action-gh-release@c062e08bd532815e2082a85e87e3ef29c3e6d191 #v2.0.8 + with: + tag_name: ${{ env.VERSION }} + name: ACCESS-RAM conda environment ${{ env.VERSION }} + generate_release_notes: true + fail_on_unmatched_files: true + files: | + ./${{ env.PACKAGE_NAME }}.tar.gz + ./${{ env.PACKAGE_NAME }}.conda-lock.yml \ No newline at end of file diff --git a/.github/workflows/update-module.yml b/.github/workflows/update-module.yml new file mode 100644 index 0000000..317120f --- /dev/null +++ b/.github/workflows/update-module.yml @@ -0,0 +1,59 @@ +name: Update common modulefiles on Gadi +on: + workflow_dispatch: + push: + branches: + - main + paths: + - 'modules/**' +jobs: + setup-deployment-env: + name: Setup Deployment Environment + runs-on: ubuntu-latest + outputs: + deployment-environments: ${{ steps.get-deployment-environment.outputs.deployment-environments }} + steps: + - name: Checkout config + uses: actions/checkout@v4 + + - name: Get environments + id: get-deployment-environment + run: echo "deployment-environments=$(jq --compact-output '.environments' ./config/deployment-environment.json)" >> $GITHUB_OUTPUT + + deploy: + runs-on: ubuntu-latest + needs: setup-deployment-env + strategy: + fail-fast: false + matrix: + deployment-environment: ${{ fromJson(needs.setup-deployment-env.outputs.deployment-environments) }} + environment: ${{ matrix.deployment-environment }} + steps: + - uses: actions/checkout@v4 + + - uses: access-nri/actions/.github/actions/setup-ssh@main + id: ssh + with: + hosts: | + ${{ secrets.HOST_DATA }} + private-key: ${{ secrets.SSH_KEY }} + + - name: Generate release modulesfiles + run: | + mkdir -p release/modules + sed 's|{{MODULE_DIR}}|'"${{ vars.MODULE_DIR }}"'|g' modules/.common > release/modules/.common + sed 's|{{APPS_DIR}}|'"${{ vars.APPS_DIR }}"'|g' modules/env.sh > release/modules/env.sh + chmod +x release/modules/env.sh + + - name: Create deployment directory on ${{ matrix.deployment-environment }} + run: | + ssh ${{ secrets.USER }}@${{ secrets.HOST_DATA }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + mkdir -p ${{ vars.MODULE_DIR }} + EOT + + - name: Copy modulefiles to Gadi + run: | + # Rsync release modulefiles + rsync -rvp -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ + release/modules/ \ + ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.MODULE_DIR }} diff --git a/README.md b/README.md index 9522c6f..37107b7 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ On all of the deployment targets, the deployed environment can be activated usin 2. Once you are a member, run the following: ```bash - module use /g/data//modules - module load access-ram3 + module use /g/data/vk83/modules + module load conda/access-ram3/ ``` You are set to run ACCESS-RAM3. @@ -44,7 +44,7 @@ New deployment environments must be created as a GitHub Environment and also hav ### Deploying locally -To deploy locally, you can use the assets created during the release. [Releases are found here](https://github.com/ACCESS-NRI/access-ram-condaenv/releases). Specifically: +To deploy locally, you can use the assets created in the release. [Releases are found here](https://github.com/ACCESS-NRI/access-ram-condaenv/releases). Specifically: -- To use the compressed environment (which doesn't require conda or python) you can run `tar -xzf access-ram.tar.gz access-ram` and then `./access-ram/bin/activate` to activate the environment. +- To use the compressed environment (which doesn't require conda or python) you can run `tar -xzf access-ram-.tar.gz access-ram` and then `./access-ram/bin/activate` to activate the environment. - To use the lockfile, you can run `micromamba create -n environment-name -f access-ram.conda-lock.yml` with an appropriate install of `micromamba`. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..d3827e7 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0 diff --git a/env-dev.yml b/env-dev.yml index 76fe706..1c240fb 100644 --- a/env-dev.yml +++ b/env-dev.yml @@ -6,12 +6,11 @@ channels: dependencies: - hypothesis - ipykernel + - mule - numpy=1.23.4 # https://stackoverflow.com/a/75148219/21024780 - - pip - python=3.10 - pytest - pytest-cov + - pre_commit - scitools-iris - - xarray - - pip: - - mule @ git+https://github.com/metomi/mule@cce4b99c7046217b1ec1192118a786636e0d8e54#subdirectory=mule + - xarray \ No newline at end of file diff --git a/env.yml b/env.yml index 438476d..e90efa0 100644 --- a/env.yml +++ b/env.yml @@ -4,10 +4,10 @@ channels: - coecms - conda-forge dependencies: + - conda-lock + - conda-pack + - mule - numpy=1.23.4 # https://stackoverflow.com/a/75148219/21024780 - - pip - python=3.10 - scitools-iris - xarray - - pip: - - mule @ git+https://github.com/metomi/mule@cce4b99c7046217b1ec1192118a786636e0d8e54#subdirectory=mule diff --git a/modules/.common b/modules/.common index eef2c7f..1a77bad 100644 --- a/modules/.common +++ b/modules/.common @@ -3,17 +3,17 @@ # This is adapted from hh5's modulefiles for activating conda environments # using modules (see /g/data/hh5/public/modules/conda/.common.v2) -module-whatis {Activate access-ram micromamba environment (https://github.com/ACCESS-NRI/access-ram-condaenv)} -conflict access-ram +module-whatis {Activate micromamba environment to run access-ram python scripts (https://github.com/ACCESS-NRI/access-ram-condaenv)} +conflict conda -# Name of this module's environment +# Name of this module's environment (e.g. conda/access-ram-VERSION) set envname [module-info name] # Get the environment variables from activate script -set payuenv [exec /bin/env -i {{MODULE_LOCATION}}/env.sh $envname] +set environment [exec /bin/env -i {{MODULE_DIR}}/env.sh $envname] # Convert the environment into module commands -set lines [split $payuenv '\n'] +set lines [split $environment '\n'] foreach line $lines { regexp {^([^=]*)=(.*)} $line -> key value diff --git a/modules/env.sh b/modules/env.sh index 0045d4a..40b631a 100755 --- a/modules/env.sh +++ b/modules/env.sh @@ -2,5 +2,5 @@ # Prints the environment variables when activating un-packed environment export PATH=/usr/bin:/bin -source {{APPS_LOCATION}}/$1/bin/activate +source {{APPS_DIR}}/$1/bin/activate /bin/env \ No newline at end of file