Merge pull request #130 from nansencenter/multi_python_versions

Support multiple python versions
nansencenter · Dec 13, 2023 · 4c9f9dd · 4c9f9dd
2 parents cb58018 + 168098e
commit 4c9f9dd
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 91 deletions.
diff --git a/.github/workflows/tests_build.yml → .github/workflows/ci.yml b/.github/workflows/tests_build.yml → .github/workflows/ci.yml
@@ -5,64 +5,65 @@ on:
     branches: ['**']
   release:
     types: [prereleased, released]
-env:
-  BASE_IMAGE_NAME: "${{ vars.DOCKER_ORG }}/geospaas:2.5.2.dev0-slim"
-  METANORM_VERSION: '4.1.0'
 jobs:
-  tests:
-    runs-on: 'ubuntu-20.04'
+  tests_and_docker_build:
+    name: 'Run tests and build docker image'
+    runs-on: 'ubuntu-latest'
+    strategy:
+      matrix:
+        python_version:
+          - '3.7'
+          - '3.8'
+          - '3.9'
+          - '3.10'
+          - '3.11'
     env:
+      BASE_IMAGE: "${{ vars.DOCKER_ORG }}/geospaas:2.5.2-python${{ matrix.python_version }}"
+      IMAGE_NAME: "${{ vars.DOCKER_ORG }}/geospaas_harvesting"
+      METANORM_VERSION: '4.1.0'
       GEOSPAAS_DB_HOST: 'db'
       GEOSPAAS_DB_USER: 'test'
-      GEOSPAAS_DB_PASSWORD: ${{ secrets.GEOSPAAS_DB_PASSWORD }}
-      IMAGE_NAME: 'harvesting_tests'
+      GEOSPAAS_DB_PASSWORD: "${{ secrets.GEOSPAAS_DB_PASSWORD }}"
+      latest: ${{ matrix.python_version == '3.11' && 'true' || '' }}
     steps:
       - name: 'Checkout repository'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: 'Create test docker network'
         run: docker network create testing
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
 
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-testing-${{ github.sha }}
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
           restore-keys: |
-            ${{ runner.os }}-buildx-testing-
+            ${{ runner.os }}-buildx-
 
       - name: Build testing image
         id: docker_build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           target: base
           build-args: |
-            BASE_IMAGE=${{ env.BASE_IMAGE_NAME }}
+            BASE_IMAGE=${{ env.BASE_IMAGE }}
             METANORM_VERSION=${{ env.METANORM_VERSION }}
           push: false
           load: true
-          tags: ${{ env.IMAGE_NAME }}
+          tags: 'harvesting_tests'
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
 
-        # Temp fix
-        # https://github.com/docker/build-push-action/issues/252
-        # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
-
       - name: 'Start testing database'
         run: /bin/bash ./scripts/start_test_database.sh
 
@@ -80,109 +81,68 @@ jobs:
              -e "GITHUB_REPOSITORY=$GITHUB_REPOSITORY"
              -e "GITHUB_RUN_ID=$GITHUB_RUN_ID"
              -e "GITHUB_TOKEN=$GITHUB_TOKEN"
-             -e "GEOSPAAS_DB_HOST=$GEOSPAAS_DB_HOST"
-             -e "GEOSPAAS_DB_USER=$GEOSPAAS_DB_USER"
-             -e "GEOSPAAS_DB_PASSWORD=$GEOSPAAS_DB_PASSWORD"
-             "${IMAGE_NAME}"
+             -e "GEOSPAAS_DB_HOST=${GEOSPAAS_DB_HOST}"
+             -e "GEOSPAAS_DB_USER=${GEOSPAAS_DB_USER}"
+             -e "GEOSPAAS_DB_PASSWORD=${GEOSPAAS_DB_PASSWORD}"
+             'harvesting_tests'
              bash -c "coverage run ./runtests.py && coverage combine"
 
       - name: 'Stop testing database'
-        run: docker stop "$GEOSPAAS_DB_HOST"
+        run: docker stop "${{ env.GEOSPAAS_DB_HOST }}"
 
-      - name: 'Install Python 3.7'
-        uses: actions/setup-python@v2
+      - name: 'Install Python 3.11'
+        if: ${{ env.latest }}
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.7'
+          python-version: '3.11'
 
       - name: 'Upload coverage to coveralls.io'
+        if: ${{ env.latest }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: pip install coveralls && coveralls --service=github
 
-  build_docker_image:
-    runs-on: 'ubuntu-20.04'
-    needs: 'tests'
-    env:
-      IMAGE_NAME: "${{ vars.DOCKER_ORG }}/geospaas_harvesting"
-    steps:
-      - name: 'Checkout repository'
-        uses: actions/checkout@v2
-
-      - name: "Extract tag name"
-        id: get_git_tag
-        run: |
-          if [[ $GITHUB_REF == refs/tags/* ]];then
-            TAG="${GITHUB_REF#refs/tags/}"
-          else
-            TAG='tmp'
-          fi
-          echo "version=$TAG" >> $GITHUB_OUTPUT
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-building-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-building-
-            ${{ runner.os }}-buildx-testing-
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USER }}
-          password: ${{ secrets.DOCKER_PASS }}
-
-      - name: "Make Docker tags"
-        id: make_docker_tags
-        run: |
-          TAGS='${{ env.IMAGE_NAME }}:${{ steps.get_git_tag.outputs.version }}'
-          if [[ '${{ github.event.action }}' == released ]];then
-            TAGS="$TAGS,${{ env.IMAGE_NAME }}:latest"
-          fi
-          echo "tags=$TAGS" >> $GITHUB_OUTPUT
-
       - name: Build docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
           build-args: |
-            BASE_IMAGE=${{ env.BASE_IMAGE_NAME }}
+            BASE_IMAGE=${{ env.BASE_IMAGE }}
             METANORM_VERSION=${{ env.METANORM_VERSION }}
-            GEOSPAAS_HARVESTING_RELEASE=${{ steps.get_git_tag.outputs.version }}
+            GEOSPAAS_HARVESTING_RELEASE=${{ github.ref_type == 'tag' && github.ref_name || '0.0.0' }}
           push: ${{ github.event_name == 'release' }}
-          tags: ${{ steps.make_docker_tags.outputs.tags }}
+          tags: |
+            ${{ env.IMAGE_NAME }}:${{ github.ref_name }}-python${{ matrix.python_version }}
+            ${{ env.latest && format('{0}:{1}', env.IMAGE_NAME, github.ref_name) || '' }}
+            ${{ env.IMAGE_NAME }}:latest-python${{ matrix.python_version }}
+            ${{ env.latest && format('{0}:latest', env.IMAGE_NAME) || '' }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
 
-        # Temp fix
+      # Temp fix
         # https://github.com/docker/build-push-action/issues/252
         # https://github.com/moby/buildkit/issues/1896
       - name: Move cache
         run: |
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
+
   publish_python_package:
     name: Build Python package and publish it as a release artifact
-    runs-on: 'ubuntu-20.04'
-    needs: 'tests'
+    runs-on: 'ubuntu-latest'
+    needs: 'tests_and_docker_build'
     if: ${{ github.event_name == 'release' }}
-    env:
-      TAG_REF: ${{ github.ref }}
     steps:
       - name: 'Checkout repository'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: 'Build Python package'
         run: >
              docker run --rm
              -v "$(pwd):/src"
-             -e "GEOSPAAS_HARVESTING_RELEASE=${TAG_REF#refs/tags/}"
-             "${BASE_IMAGE_NAME}"
+             -e "GEOSPAAS_HARVESTING_RELEASE=${{ github.ref_name }}"
+             "${{ vars.DOCKER_ORG }}/geospaas:latest"
              python setup.py sdist bdist_wheel
 
       - name: 'Deploy package to the Github release'

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=nansencenter/geospaas:latest-slim
+ARG BASE_IMAGE=nansencenter/geospaas:latest
 
 FROM ${BASE_IMAGE} as base