From deb3defc21d03c491f7a0fa6445795330b5162c0 Mon Sep 17 00:00:00 2001 From: Firelight Flagboy Date: Mon, 13 Nov 2023 16:26:21 +0100 Subject: [PATCH] Rework `docker-testbed` workflow - Rename the workflow from `publish-testbed` to `docker-testbed` To harmonize with the yet to exist workflow to build the docker image for the parsec backend. - Prevent concurrency run. - Use the gihtub action `docker/setup-buildx-action` to setup buildkit engine for docker, that provide support for the cache system & multi platforms builds. - Use the github action `docker/metadata-actions`: - Generate the github tags for us. - Handle the image name (would could setup to push to `ghcr.io` & `docker.io` in the future). - Use the github action `docker/build-publish-actions`: - Simplify configuring the docker build phase. - Leverage the github actions cache. - Would allow to build for multiple platforms. - Allow to simpli push the generate image with its multiple tags. - Rework the test part, mainly split into 3 steps instead of 1. --- .github/workflows/docker-testbed.yml | 130 ++++++++++++++++++ .github/workflows/publish-testbed.yml | 88 ------------ .../testbed-server/testbed-server.dockerfile | 2 +- 3 files changed, 131 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/docker-testbed.yml delete mode 100644 .github/workflows/publish-testbed.yml diff --git a/.github/workflows/docker-testbed.yml b/.github/workflows/docker-testbed.yml new file mode 100644 index 00000000000..35f51cd3796 --- /dev/null +++ b/.github/workflows/docker-testbed.yml @@ -0,0 +1,130 @@ +name: Build & Publish docker testbed + +on: + # This job must be manually triggered to publish a new version usable from + # other CI runs. + # (see https://github.com/Scille/parsec-cloud/pkgs/container/parsec-cloud%2Fparsec-testbed-server) + workflow_dispatch: + pull_request: + paths: + # Testbed code also depends on `libparsec/**`, but this code change very often + # and we consider the server tests are good enough on this part. + - server/tests/scripts/run_testbed_server.py + - server/packaging/testbed-server/** + - .github/workflows/docker-testbed.yml + push: + branches: + - master + paths: + # Testbed code also depends on `libparsec/**`, but this code change very often + # and we consider the server tests are good enough on this part. + - server/tests/scripts/run_testbed_server.py + - server/packaging/testbed-server/** + - .github/workflows/docker-testbed.yml + +permissions: + contents: write + packages: write + +# We set `concurrency` to prevent having this workflow being run on code that is not up-to-date on a PR (a user make multiple push in a quick manner). +# But on the main branch, we don't want that behavior. +# Having the workflow run on each merge commit is something we would like, that could help us where a regression was made and missed by previous checks. +concurrency: + group: docker-testbed-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + docker-testbed: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # pin v4.1.1 + timeout-minutes: 5 + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Log in to the Github Container registry + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Get current version + id: version + shell: python + run: | + from version_updater import TOOLS_VERSION, Tool + + print(f"current={TOOLS_VERSION[Tool.Parsec]}") + working-directory: misc + + - name: Generate build metadata + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + id: metadata + with: + images: + ghcr.io/scille/parsec-cloud/parsec-testbed-server + tags: | + type=raw,value=${{ steps.version.outputs.current }}+{{ date 'YYYYMMDD' }}.sha.{{ sha }} + flavor: | + latest=${{ github.event_name == 'workflow_dispatch' }} + + - name: Build and export to Docker + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + id: build + with: + context: . + file: server/packaging/testbed-server/testbed-server.dockerfile + load: true + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + push: false + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Start docker test container + id: test-container + run: | + ( + echo -n "id="; + docker run --detach --publish 6777:6777 --rm --name=parsec-testbed-server ${{ steps.build.outputs.imageid }} + ) | tee $GITHUB_OUTPUT + + - name: Test docker image + run: | + import time + from urllib.request import Request, urlopen + + r = Request('http://127.0.0.1:6777/testbed/new/empty', method='POST') + for i in range(10): + try: + urlopen(r) + except Exception as exc: + print(f'Try {i + 1}/10: {exc}') + time.sleep(1) + continue + else: + break + else: + raise SystemExit('Cannot connect to testbed server :(') + shell: python + + - name: Stop docker test container + run: | + docker container stop ${{ steps.test-container.outputs.id }} + docker container rm ${{ steps.test-container.outputs.id }} + + - name: Build and publish + if: github.event_name == 'workflow_dispatch' + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: . + file: server/packaging/testbed-server/testbed-server.dockerfile + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + push: true + cache-from: type=gha diff --git a/.github/workflows/publish-testbed.yml b/.github/workflows/publish-testbed.yml deleted file mode 100644 index 4bd51800749..00000000000 --- a/.github/workflows/publish-testbed.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Publish docker testbed - -on: - # This job must be manually triggered to publish a new version usable from - # other CI runs. - # (see https://github.com/Scille/parsec-cloud/pkgs/container/parsec-cloud%2Fparsec-testbed-server) - workflow_dispatch: - pull_request: - paths: - # Testbed code also depends on `libparsec/**`, but this code change very often - # and we consider the server tests are good enough on this part. - - server/tests/scripts/run_testbed_server.py - - server/packaging/testbed-server/** - - .github/workflows/publish-testbed.yml - push: - branches: - - master - paths: - # Testbed code also depends on `libparsec/**`, but this code change very often - # and we consider the server tests are good enough on this part. - - server/tests/scripts/run_testbed_server.py - - server/packaging/testbed-server/** - - .github/workflows/publish-testbed.yml - -permissions: - contents: write - packages: write - -jobs: - publish-testbed: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # pin v4.1.1 - timeout-minutes: 5 - - - name: Log in to the Container registry - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build docker image - run: bash server/packaging/testbed-server/build.sh 2>&1 - env: - WRITE_ENV: 1 - timeout-minutes: 20 - - - name: Test docker image - run: | - set -ex - - source parsec-testbed.env - - docker run --publish 6777:6777 --rm --name=parsec-testbed-server $PREFIX/parsec-testbed-server:$UNIQ_TAG & - - python -c " - import time - from urllib.request import Request, urlopen - - r = Request('http://127.0.0.1:6777/testbed/new/empty', method='POST') - for i in range(10): - try: - urlopen(r) - except Exception as exc: - print(f'Try {i + 1}/10: {exc}') - time.sleep(1) - continue - else: - break - else: - raise SystemExit('Cannot connect to testbed server :(') - " - - kill %1 - - - name: Publish docker image - if: github.event_name == 'workflow_dispatch' - run: | - set -ex - - source parsec-testbed.env - - for tag in $UNIQ_TAG; do - echo "Pushing tag \`$tag\` to \`$PREFIX\`" - docker push $PREFIX/parsec-testbed-server:$tag - done - timeout-minutes: 5 diff --git a/server/packaging/testbed-server/testbed-server.dockerfile b/server/packaging/testbed-server/testbed-server.dockerfile index f9ad95325e1..d80a72faec7 100644 --- a/server/packaging/testbed-server/testbed-server.dockerfile +++ b/server/packaging/testbed-server/testbed-server.dockerfile @@ -18,7 +18,7 @@ ADD --link \ Cargo.toml \ make.py \ server/packaging/testbed-server/in-docker-build.sh \ - . + ./ ADD --link libparsec/ libparsec/ ADD --link server/ server/ ADD --link bindings/ bindings/