From 4d6384ce7dd11fb131d7445a3f0c07c3844cda76 Mon Sep 17 00:00:00 2001
From: Adrien Perrin <adrien.perrin@nersc.no>
Date: Mon, 20 Nov 2023 11:37:28 +0100
Subject: [PATCH 1/4] update actions to support multiple python versions

---
 .github/workflows/{tests_build.yml => ci.yml} | 139 ++++++------------
 1 file changed, 49 insertions(+), 90 deletions(-)
 rename .github/workflows/{tests_build.yml => ci.yml} (51%)

diff --git a/.github/workflows/tests_build.yml b/.github/workflows/ci.yml
similarity index 51%
rename from .github/workflows/tests_build.yml
rename to .github/workflows/ci.yml
index 70442f78..cde58272 100644
--- a/.github/workflows/tests_build.yml
+++ b/.github/workflows/ci.yml
@@ -5,64 +5,64 @@ on:
     branches: ['**']
   release:
     types: [prereleased, released]
-env:
-  BASE_IMAGE_NAME: "${{ vars.DOCKER_ORG }}/geospaas:2.5.2.dev0-slim"
-  METANORM_VERSION: '4.1.0'
 jobs:
-  tests:
-    runs-on: 'ubuntu-20.04'
+  tests_and_docker_build:
+    name: 'Run tests and build docker image'
+    runs-on: 'ubuntu-latest'
+    strategy:
+      matrix:
+        version:
+          - {'python': '3.7', 'latest': false}
+          - {'python': '3.8', 'latest': false}
+          - {'python': '3.9', 'latest': false}
+          - {'python': '3.10', 'latest': false}
+          - {'python': '3.11', 'latest': true}
     env:
+      BASE_IMAGE: "${{ vars.DOCKER_ORG }}/geospaas:0.0.2-slim-python${{ matrix.version.python }}"
+      IMAGE_NAME: "${{ vars.DOCKER_ORG }}/geospaas_harvesting"
+      METANORM_VERSION: '4.1.0'
       GEOSPAAS_DB_HOST: 'db'
       GEOSPAAS_DB_USER: 'test'
-      GEOSPAAS_DB_PASSWORD: ${{ secrets.GEOSPAAS_DB_PASSWORD }}
-      IMAGE_NAME: 'harvesting_tests'
+      GEOSPAAS_DB_PASSWORD: "${{ secrets.GEOSPAAS_DB_PASSWORD }}"
     steps:
       - name: 'Checkout repository'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: 'Create test docker network'
         run: docker network create testing
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
 
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-testing-${{ github.sha }}
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
           restore-keys: |
-            ${{ runner.os }}-buildx-testing-
+            ${{ runner.os }}-buildx-
 
       - name: Build testing image
         id: docker_build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           target: base
           build-args: |
-            BASE_IMAGE=${{ env.BASE_IMAGE_NAME }}
+            BASE_IMAGE=${{ env.BASE_IMAGE }}
             METANORM_VERSION=${{ env.METANORM_VERSION }}
           push: false
           load: true
-          tags: ${{ env.IMAGE_NAME }}
+          tags: 'harvesting_tests'
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
 
-        # Temp fix
-        # https://github.com/docker/build-push-action/issues/252
-        # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
-
       - name: 'Start testing database'
         run: /bin/bash ./scripts/start_test_database.sh
 
@@ -80,85 +80,45 @@ jobs:
              -e "GITHUB_REPOSITORY=$GITHUB_REPOSITORY"
              -e "GITHUB_RUN_ID=$GITHUB_RUN_ID"
              -e "GITHUB_TOKEN=$GITHUB_TOKEN"
-             -e "GEOSPAAS_DB_HOST=$GEOSPAAS_DB_HOST"
-             -e "GEOSPAAS_DB_USER=$GEOSPAAS_DB_USER"
-             -e "GEOSPAAS_DB_PASSWORD=$GEOSPAAS_DB_PASSWORD"
-             "${IMAGE_NAME}"
+             -e "GEOSPAAS_DB_HOST=${GEOSPAAS_DB_HOST}"
+             -e "GEOSPAAS_DB_USER=${GEOSPAAS_DB_USER}"
+             -e "GEOSPAAS_DB_PASSWORD=${GEOSPAAS_DB_PASSWORD}"
+             'harvesting_tests'
              bash -c "coverage run ./runtests.py && coverage combine"
 
       - name: 'Stop testing database'
-        run: docker stop "$GEOSPAAS_DB_HOST"
+        run: docker stop "${{ env.GEOSPAAS_DB_HOST }}"
 
-      - name: 'Install Python 3.7'
-        uses: actions/setup-python@v2
+      - name: 'Install Python 3.11'
+        if: ${{ matrix.version.latest }}
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.7'
+          python-version: '3.11'
 
       - name: 'Upload coverage to coveralls.io'
+        if: ${{ matrix.version.latest }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: pip install coveralls && coveralls --service=github
 
-  build_docker_image:
-    runs-on: 'ubuntu-20.04'
-    needs: 'tests'
-    env:
-      IMAGE_NAME: "${{ vars.DOCKER_ORG }}/geospaas_harvesting"
-    steps:
-      - name: 'Checkout repository'
-        uses: actions/checkout@v2
-
-      - name: "Extract tag name"
-        id: get_git_tag
-        run: |
-          if [[ $GITHUB_REF == refs/tags/* ]];then
-            TAG="${GITHUB_REF#refs/tags/}"
-          else
-            TAG='tmp'
-          fi
-          echo "version=$TAG" >> $GITHUB_OUTPUT
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-building-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-building-
-            ${{ runner.os }}-buildx-testing-
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USER }}
-          password: ${{ secrets.DOCKER_PASS }}
-
-      - name: "Make Docker tags"
-        id: make_docker_tags
-        run: |
-          TAGS='${{ env.IMAGE_NAME }}:${{ steps.get_git_tag.outputs.version }}'
-          if [[ '${{ github.event.action }}' == released ]];then
-            TAGS="$TAGS,${{ env.IMAGE_NAME }}:latest"
-          fi
-          echo "tags=$TAGS" >> $GITHUB_OUTPUT
-
       - name: Build docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
           build-args: |
-            BASE_IMAGE=${{ env.BASE_IMAGE_NAME }}
+            BASE_IMAGE=${{ env.BASE_IMAGE }}
             METANORM_VERSION=${{ env.METANORM_VERSION }}
-            GEOSPAAS_HARVESTING_RELEASE=${{ steps.get_git_tag.outputs.version }}
+            GEOSPAAS_HARVESTING_RELEASE=${{ github.ref_type == 'tag' && github.ref_name || '0.0.0' }}
           push: ${{ github.event_name == 'release' }}
-          tags: ${{ steps.make_docker_tags.outputs.tags }}
+          tags: |
+            ${{ env.IMAGE_NAME }}:${{ github.ref_name }}-python${{ matrix.version.python }}
+            ${{ matrix.version.latest && format('{0}:{1}', env.IMAGE_NAME, github.ref_name) || '' }}
+            ${{ env.IMAGE_NAME }}:latest-python${{ matrix.version.python }}
+            ${{ matrix.version.latest && format('{0}:latest', env.IMAGE_NAME) || '' }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
 
-        # Temp fix
+      # Temp fix
         # https://github.com/docker/build-push-action/issues/252
         # https://github.com/moby/buildkit/issues/1896
       - name: Move cache
@@ -166,23 +126,22 @@ jobs:
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
+
   publish_python_package:
     name: Build Python package and publish it as a release artifact
-    runs-on: 'ubuntu-20.04'
-    needs: 'tests'
+    runs-on: 'ubuntu-latest'
+    needs: 'tests_and_docker_build'
     if: ${{ github.event_name == 'release' }}
-    env:
-      TAG_REF: ${{ github.ref }}
     steps:
       - name: 'Checkout repository'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: 'Build Python package'
         run: >
              docker run --rm
              -v "$(pwd):/src"
-             -e "GEOSPAAS_HARVESTING_RELEASE=${TAG_REF#refs/tags/}"
-             "${BASE_IMAGE_NAME}"
+             -e "GEOSPAAS_HARVESTING_RELEASE=${{ github.ref_name }}"
+             "${{ vars.DOCKER_ORG }}/geospaas:latest-slim"
              python setup.py sdist bdist_wheel
 
       - name: 'Deploy package to the Github release'

From 3e12fa9e9f55be6dfb473cf881e5d6b22f987dab Mon Sep 17 00:00:00 2001
From: Adrien Perrin <adrien.perrin@nersc.no>
Date: Thu, 7 Dec 2023 17:06:30 +0100
Subject: [PATCH 2/4] stop using slim images

---
 .github/workflows/ci.yml | 4 ++--
 Dockerfile               | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cde58272..b3a5b791 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -18,7 +18,7 @@ jobs:
           - {'python': '3.10', 'latest': false}
           - {'python': '3.11', 'latest': true}
     env:
-      BASE_IMAGE: "${{ vars.DOCKER_ORG }}/geospaas:0.0.2-slim-python${{ matrix.version.python }}"
+      BASE_IMAGE: "${{ vars.DOCKER_ORG }}/geospaas:0.0.4-python${{ matrix.version.python }}"
       IMAGE_NAME: "${{ vars.DOCKER_ORG }}/geospaas_harvesting"
       METANORM_VERSION: '4.1.0'
       GEOSPAAS_DB_HOST: 'db'
@@ -141,7 +141,7 @@ jobs:
              docker run --rm
              -v "$(pwd):/src"
              -e "GEOSPAAS_HARVESTING_RELEASE=${{ github.ref_name }}"
-             "${{ vars.DOCKER_ORG }}/geospaas:latest-slim"
+             "${{ vars.DOCKER_ORG }}/geospaas:latest"
              python setup.py sdist bdist_wheel
 
       - name: 'Deploy package to the Github release'
diff --git a/Dockerfile b/Dockerfile
index 1f89b856..5c93698d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=nansencenter/geospaas:latest-slim
+ARG BASE_IMAGE=nansencenter/geospaas:latest
 
 FROM ${BASE_IMAGE} as base
 

From 35781f643cb2c7784a9e33017630b26708d9bfb6 Mon Sep 17 00:00:00 2001
From: Adrien Perrin <adrien.perrin@nersc.no>
Date: Tue, 12 Dec 2023 10:57:24 +0100
Subject: [PATCH 3/4] manage latest python version more elegantly

---
 .github/workflows/ci.yml | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b3a5b791..80b7db21 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -11,19 +11,20 @@ jobs:
     runs-on: 'ubuntu-latest'
     strategy:
       matrix:
-        version:
-          - {'python': '3.7', 'latest': false}
-          - {'python': '3.8', 'latest': false}
-          - {'python': '3.9', 'latest': false}
-          - {'python': '3.10', 'latest': false}
-          - {'python': '3.11', 'latest': true}
+        python_version:
+          - '3.7'
+          - '3.8'
+          - '3.9'
+          - '3.10'
+          - '3.11'
     env:
-      BASE_IMAGE: "${{ vars.DOCKER_ORG }}/geospaas:0.0.4-python${{ matrix.version.python }}"
+      BASE_IMAGE: "${{ vars.DOCKER_ORG }}/geospaas:0.0.4-python${{ matrix.python_version }}"
       IMAGE_NAME: "${{ vars.DOCKER_ORG }}/geospaas_harvesting"
       METANORM_VERSION: '4.1.0'
       GEOSPAAS_DB_HOST: 'db'
       GEOSPAAS_DB_USER: 'test'
       GEOSPAAS_DB_PASSWORD: "${{ secrets.GEOSPAAS_DB_PASSWORD }}"
+      latest: ${{ matrix.python_version == '3.11' && 'true' || '' }}
     steps:
       - name: 'Checkout repository'
         uses: actions/checkout@v4
@@ -90,13 +91,13 @@ jobs:
         run: docker stop "${{ env.GEOSPAAS_DB_HOST }}"
 
       - name: 'Install Python 3.11'
-        if: ${{ matrix.version.latest }}
+        if: ${{ env.latest }}
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
       - name: 'Upload coverage to coveralls.io'
-        if: ${{ matrix.version.latest }}
+        if: ${{ env.latest }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: pip install coveralls && coveralls --service=github
@@ -111,10 +112,10 @@ jobs:
             GEOSPAAS_HARVESTING_RELEASE=${{ github.ref_type == 'tag' && github.ref_name || '0.0.0' }}
           push: ${{ github.event_name == 'release' }}
           tags: |
-            ${{ env.IMAGE_NAME }}:${{ github.ref_name }}-python${{ matrix.version.python }}
-            ${{ matrix.version.latest && format('{0}:{1}', env.IMAGE_NAME, github.ref_name) || '' }}
-            ${{ env.IMAGE_NAME }}:latest-python${{ matrix.version.python }}
-            ${{ matrix.version.latest && format('{0}:latest', env.IMAGE_NAME) || '' }}
+            ${{ env.IMAGE_NAME }}:${{ github.ref_name }}-python${{ matrix.python_version }}
+            ${{ env.latest && format('{0}:{1}', env.IMAGE_NAME, github.ref_name) || '' }}
+            ${{ env.IMAGE_NAME }}:latest-python${{ matrix.python_version }}
+            ${{ env.latest && format('{0}:latest', env.IMAGE_NAME) || '' }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
 

From 168098eeae4d9e543a5ef17da2398ec9875cdfb9 Mon Sep 17 00:00:00 2001
From: Adrien Perrin <adrien.perrin@nersc.no>
Date: Tue, 12 Dec 2023 14:36:37 +0100
Subject: [PATCH 4/4] update base image version to 2.5.2

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 80b7db21..e28b0a44 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -18,7 +18,7 @@ jobs:
           - '3.10'
           - '3.11'
     env:
-      BASE_IMAGE: "${{ vars.DOCKER_ORG }}/geospaas:0.0.4-python${{ matrix.python_version }}"
+      BASE_IMAGE: "${{ vars.DOCKER_ORG }}/geospaas:2.5.2-python${{ matrix.python_version }}"
       IMAGE_NAME: "${{ vars.DOCKER_ORG }}/geospaas_harvesting"
       METANORM_VERSION: '4.1.0'
       GEOSPAAS_DB_HOST: 'db'