From 17926da987e9be7c5cabe39a6dbb1e79bebe0cb1 Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Tue, 25 Jul 2023 12:20:27 +0200 Subject: [PATCH 01/10] CI/CD: publish docker image in GHCR --- .github/actions/build/action.yml | 6 +- .github/workflows/delivery.yml | 90 +++++++++++++++---- .../workflows/status-checks-dockerfile.yml | 4 +- Dockerfile | 31 +++---- 4 files changed, 96 insertions(+), 35 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 284513b8..997e39d0 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -23,13 +23,13 @@ runs: - name: Determine artifact output filename id: artifact-name-generator - run: echo "artifact-name=irma-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.os == 'windows' && '.exe' || '' }}" >> $GITHUB_OUTPUT + run: echo "artifact-name=irma-${{ inputs.os }}-${{ inputs.arch }}${{ inputs.os == 'windows' && '.exe' || '' }}" >> $GITHUB_OUTPUT shell: bash - name: Build run: go build -v -a -ldflags '-extldflags "-static"' -o ${{ steps.artifact-name-generator.outputs.artifact-name }} ./irma shell: bash env: - GOOS: ${{ matrix.os }} - GOARCH: ${{ matrix.arch }} + GOOS: ${{ inputs.os }} + GOARCH: ${{ inputs.arch }} CGO_ENABLED: "0" diff --git a/.github/workflows/delivery.yml b/.github/workflows/delivery.yml index e9bcf211..ddfd4036 100644 --- a/.github/workflows/delivery.yml +++ b/.github/workflows/delivery.yml @@ -2,17 +2,25 @@ name: Delivery on: push: - tags: [ v* ] + branches: [ master ] + release: + types: [ created, edited ] permissions: contents: write + packages: write + +# Disable concurrency to prevent that images are tagged in the wrong order. +concurrency: + group: delivery jobs: - ensure-release-present: + prepare: runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + outputs: + is-head-master: ${{ steps.is-head-master.outcome == 'success' }} + is-latest-release: ${{ steps.is-latest-release.outcome == 'success' }} steps: - uses: actions/checkout@v3 with: @@ -23,30 +31,73 @@ jobs: run: git branch --contains ${{ github.sha }} | grep -x "* master" shell: bash + - name: Check whether this event is the HEAD of master + continue-on-error: true + id: is-head-master + run: git rev-parse HEAD | grep -x ${{ github.sha }} + shell: bash + - uses: actions/checkout@v3 - name: Check whether version.go contains the new version number + if: github.event_name == 'release' run: cat version.go | grep ${GITHUB_REF_NAME:1} shell: bash - name: Check whether CHANGELOG.md contains the new version number + if: github.event_name == 'release' run: cat CHANGELOG.md | grep "\[${GITHUB_REF_NAME:1}\]" shell: bash - - name: Check whether release is present - id: release-present - run: gh release view ${{ github.ref_name }} + - name: Check whether the release is latest continue-on-error: true + id: is-latest-release + if: github.event_name == 'release' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release view --json tagName --jq .tagName | grep -x ${{ github.event.release.tag_name }} + shell: bash + + build-docker-image: + runs-on: ubuntu-latest + needs: prepare + steps: + - uses: actions/checkout@v3 - - name: Make new release if necessary - if: steps.release-present.outcome == 'failure' - run: gh release create ${{ github.ref_name }} -t "${{ github.ref_name }}" -n "Check CHANGELOG.md in repository." + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull Docker image + continue-on-error: true + id: docker-pull + run: docker pull ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} + + - name: Build Docker image (if necessary) + if: steps.docker-pull.outcome == 'failure' + run: docker build -t ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} . + + - name: Tag Docker image (edge) + if: needs.prepare.outputs.is-head-master == 'true' + run: docker tag ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} ghcr.io/${{ github.repository_owner }}/irma:edge + + - name: Tag Docker image (version) + if: github.event_name == 'release' + run: docker tag ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} ghcr.io/${{ github.repository_owner }}/irma:${{ github.event.release.tag_name }} + + - name: Tag Docker image (latest) + if: needs.prepare.outputs.is-latest-release == 'true' + run: docker tag ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} ghcr.io/${{ github.repository_owner }}/irma:latest + + - name: Push Docker image + run: docker push --all-tags ghcr.io/${{ github.repository_owner }}/irma build-release-artifact: - needs: ensure-release-present + needs: prepare runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} strategy: matrix: os: [ linux, darwin, windows ] @@ -65,6 +116,15 @@ jobs: with: os: ${{ matrix.os }} arch: ${{ matrix.arch }} - + - name: Upload artifact - run: gh release upload ${{ github.ref_name }} ${{ steps.build.outputs.artifact-name }} + uses: actions/upload-artifact@v3 + with: + name: irma-${{ matrix.os }}-${{ matrix.arch }} + path: ${{ steps.build.outputs.artifact-name }} + + - name: Upload artifact to release + if: github.event_name == 'release' && github.event.action == 'created' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release upload ${{ github.event.release.tag_name }} ${{ steps.build.outputs.artifact-name }} diff --git a/.github/workflows/status-checks-dockerfile.yml b/.github/workflows/status-checks-dockerfile.yml index 8df2b7e7..7bc35338 100644 --- a/.github/workflows/status-checks-dockerfile.yml +++ b/.github/workflows/status-checks-dockerfile.yml @@ -34,7 +34,7 @@ jobs: - uses: actions/checkout@v3 - name: Build Dockerfile - run: docker build -t privacybydesign/irma:edge --build-arg BASE_IMAGE=${{ matrix.image }} . + run: docker build -t ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} --build-arg BASE_IMAGE=${{ matrix.image }} . - name: Test Docker image - run: docker run privacybydesign/irma:edge version + run: docker run ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} version diff --git a/Dockerfile b/Dockerfile index aec407c8..d4fc9b57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,4 @@ -# Use variable base image, such that we can also build for other base images, like alpine. -ARG BASE_IMAGE=debian:stable-slim - -FROM golang:1 as build +FROM golang:1-alpine as build # Set build environment ENV CGO_ENABLED=0 @@ -11,23 +8,27 @@ COPY . /irmago WORKDIR /irmago RUN go build -a -ldflags '-extldflags "-static"' -o "/bin/irma" ./irma -FROM $BASE_IMAGE +# Create application user +RUN adduser -D -u 1000 -g irma irma -# The amazonlinux image does not include adduser, so we have to install this first. -RUN if grep -q -E 'Amazon Linux' /etc/os-release; then yum install -y shadow-utils; fi +# Start building the final image +FROM scratch -# Add application user -RUN adduser --disabled-password --gecos '' irma || adduser irma +# Copy binary from build stage +COPY --from=build /bin/irma /bin/irma -# The debian image does not include ca-certificates, so we have to install this first. -RUN if which apt-get &> /dev/null; then apt-get update && apt-get install -y ca-certificates; fi +# Add TLS root certificates +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY --from=build /bin/irma /usr/local/bin/irma +# Ensure the application user and group is set +COPY --from=build /etc/passwd /etc/passwd +COPY --from=build /etc/group /etc/group +COPY --from=build --chown=irma:irma /home/irma/ /home/irma/ # Switch to application user USER irma -# Include schemes in the Docker image to speed up the start-up time. -RUN irma scheme download +# Include schemes in the Docker image to speed up the start-up time +RUN ["/bin/irma", "scheme", "download"] -ENTRYPOINT ["irma"] +ENTRYPOINT ["/bin/irma"] From ab0e9121a416a0169fe76d9704e920cafcf13af6 Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Tue, 25 Jul 2023 12:33:16 +0200 Subject: [PATCH 02/10] Chore: remove superfluous status-checks-dockerfile workflow --- .../workflows/status-checks-dockerfile.yml | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 .github/workflows/status-checks-dockerfile.yml diff --git a/.github/workflows/status-checks-dockerfile.yml b/.github/workflows/status-checks-dockerfile.yml deleted file mode 100644 index 7bc35338..00000000 --- a/.github/workflows/status-checks-dockerfile.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Workflow to check whether changes to master concerning the Dockerfile fulfill all requirements. -name: Status checks (Dockerfile) - -on: - push: - branches: [ master ] - paths: - - Dockerfile - pull_request: - paths: - - Dockerfile - schedule: - # Run every monday on 9:00 in the morning (UTC). - - cron: "0 9 * * 1" - # Make it possible to trigger the checks manually. - workflow_dispatch: - -jobs: - # Building the Dockerfile includes downloading the IRMA schemes. - # Therefore, we only run one check at the time, and we put a limit on the event types triggering this job. - docker-build-all: - runs-on: ubuntu-latest - strategy: - max-parallel: 1 - matrix: - # busybox is not working yet. - image: - - "debian:stable" - - "alpine:latest" - - "ubuntu:latest" - - "centos:latest" - - "amazonlinux:latest" - steps: - - uses: actions/checkout@v3 - - - name: Build Dockerfile - run: docker build -t ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} --build-arg BASE_IMAGE=${{ matrix.image }} . - - - name: Test Docker image - run: docker run ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} version From a59810a6da210a0150aac4e9e5e688f29f3e1584 Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Tue, 25 Jul 2023 12:36:49 +0200 Subject: [PATCH 03/10] Fix: release event type created is not always triggered --- .github/workflows/delivery.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/delivery.yml b/.github/workflows/delivery.yml index ddfd4036..fe7a1f65 100644 --- a/.github/workflows/delivery.yml +++ b/.github/workflows/delivery.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] release: - types: [ created, edited ] + types: [ published, edited ] permissions: contents: write @@ -124,7 +124,7 @@ jobs: path: ${{ steps.build.outputs.artifact-name }} - name: Upload artifact to release - if: github.event_name == 'release' && github.event.action == 'created' + if: github.event_name == 'release' && github.event.action == 'published' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: gh release upload ${{ github.event.release.tag_name }} ${{ steps.build.outputs.artifact-name }} From 56e71d7b52a9707bc1a9526126eef4d2a8437f86 Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Tue, 25 Jul 2023 12:47:25 +0200 Subject: [PATCH 04/10] Improvement: delivery workflow does not need to run on release edit --- .github/workflows/delivery.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/delivery.yml b/.github/workflows/delivery.yml index fe7a1f65..fc839c95 100644 --- a/.github/workflows/delivery.yml +++ b/.github/workflows/delivery.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] release: - types: [ published, edited ] + types: [ published ] permissions: contents: write @@ -124,7 +124,7 @@ jobs: path: ${{ steps.build.outputs.artifact-name }} - name: Upload artifact to release - if: github.event_name == 'release' && github.event.action == 'published' + if: github.event_name == 'release' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: gh release upload ${{ github.event.release.tag_name }} ${{ steps.build.outputs.artifact-name }} From aed78506c36e3ade40185d5e9b0aae010c40fe4a Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Tue, 25 Jul 2023 14:46:07 +0200 Subject: [PATCH 05/10] Improvement: prevent unused containers being stored --- .github/workflows/delivery.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/delivery.yml b/.github/workflows/delivery.yml index fc839c95..a873bf7a 100644 --- a/.github/workflows/delivery.yml +++ b/.github/workflows/delivery.yml @@ -4,6 +4,7 @@ on: push: branches: [ master ] release: + # Note: a current limitation is that when a release is edited after publication, then the Docker tags are not automatically updated. types: [ published ] permissions: @@ -71,13 +72,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull Docker image - continue-on-error: true - id: docker-pull - run: docker pull ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} - - - name: Build Docker image (if necessary) - if: steps.docker-pull.outcome == 'failure' + - name: Build Docker image run: docker build -t ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} . - name: Tag Docker image (edge) @@ -92,8 +87,17 @@ jobs: if: needs.prepare.outputs.is-latest-release == 'true' run: docker tag ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} ghcr.io/${{ github.repository_owner }}/irma:latest - - name: Push Docker image - run: docker push --all-tags ghcr.io/${{ github.repository_owner }}/irma + - name: Push Docker image (edge) + if: needs.prepare.outputs.is-head-master == 'true' + run: docker push ghcr.io/${{ github.repository_owner }}/irma:edge + + - name: Push Docker image (version) + if: github.event_name == 'release' + run: docker push ghcr.io/${{ github.repository_owner }}/irma:${{ github.event.release.tag_name }} + + - name: Tag Docker image (latest) + if: needs.prepare.outputs.is-latest-release == 'true' + run: docker push ghcr.io/${{ github.repository_owner }}/irma:latest build-release-artifact: needs: prepare From eedcea655b9767c00d6c41414854353abe086b38 Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Tue, 25 Jul 2023 14:46:35 +0200 Subject: [PATCH 06/10] Docs: add container registry instructions --- README.md | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index b4952222..99991f1c 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ Technical documentation of all components of `irmago` and more can be found at https://irma.app/docs. -## Running +## Running (development) -The easiest way to run the `irma` command line tool is using Docker. +The easiest way to run the `irma` command line tool for development purposes is using Docker. docker-compose run irma @@ -30,30 +30,38 @@ You can run the `irma keyshare` services locally using the test configuration in docker-compose run -p 8081:8081 irma keyshare myirmaserver -c ./testdata/configurations/myirmaserver.yml ## Installing +### Using Go +To install the latest released version of the `irma` command line tool using Go, you do the following. - git clone https://github.com/privacybydesign/irmago + go install github.com/privacybydesign/irmago/irma@latest -`irmago` and its subpackages use Go modules for their dependencies. The `go` command will automatically download dependencies when needed. +You can also specify an exact version. You should replace `v0.0.0` with the desired version number. + + go install github.com/privacybydesign/irmago/irma@v0.0.0 -To install the `irma` command line tool: +### Using a container +If you want a container image of the `irma` command line tool, then you can use our `ghcr.io/privacybydesign/irma` image. - go install ./irma + docker run ghcr.io/privacybydesign/irma:latest -You can also include the `irma` command line tool in a Docker image, using a base image of your choice. The default base image is Debian's `stable-slim`. +The images are tagged in the following way: +- `latest`: latest released version of `irma` +- `edge`: HEAD of the main development branch (`master`) +- `v0.0.0`: `irma` version (replace `v0.0.0` with the desired version number) - docker build --build-arg BASE_IMAGE=alpine -t privacybydesign/irma:edge . +When you build for production, we recommend you to use the [latest release](https://github.com/privacybydesign/irmago/releases/latest). -When you build for production, we recommend you to build the [latest release](https://github.com/privacybydesign/irmago/releases/latest). You should replace `v0.0.0` with the latest version number. - - docker build -t privacybydesign/irma https://github.com/privacybydesign/irmago.git#v0.0.0 - -In case you want to build `v0.8.0` or lower, then you should do some extra steps. The `Dockerfile` was not part of the repository at that time. +In case you want to use `v0.12.6` or lower, then you should build it yourself. VERSION=v0.8.0 git checkout $VERSION git checkout master -- Dockerfile docker build -t privacybydesign/irma:$VERSION . +### Using pre-compiled binaries +You can find pre-compiled binaries of the `irma` command line tool on the [GitHub release page](https://github.com/privacybydesign/irmago/releases). +We recommend you to use the [latest release](https://github.com/privacybydesign/irmago/releases/latest). + ## Running the unit tests Some of the unit tests connect to locally running external services, namely PostgreSQL and an SMTP server running at port 1025. These need to be up and running before these tests can be executed. This can either be done using `docker-compose` or by following the instructions below to install the services manually. From f449f608eb1e0fddfb54b9c830dd5c122f6c56ba Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Tue, 25 Jul 2023 14:51:10 +0200 Subject: [PATCH 07/10] Refactor: remove superfluous whitespace --- .github/workflows/delivery.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/delivery.yml b/.github/workflows/delivery.yml index a873bf7a..7a8da789 100644 --- a/.github/workflows/delivery.yml +++ b/.github/workflows/delivery.yml @@ -120,7 +120,7 @@ jobs: with: os: ${{ matrix.os }} arch: ${{ matrix.arch }} - + - name: Upload artifact uses: actions/upload-artifact@v3 with: From 00783c51edfaa1f9ac87d768d0d1688b2c581b87 Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Tue, 25 Jul 2023 15:03:26 +0200 Subject: [PATCH 08/10] Chore: re-add status-checks-dockerfile workflow --- .../workflows/status-checks-dockerfile.yml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/status-checks-dockerfile.yml diff --git a/.github/workflows/status-checks-dockerfile.yml b/.github/workflows/status-checks-dockerfile.yml new file mode 100644 index 00000000..5c9e8216 --- /dev/null +++ b/.github/workflows/status-checks-dockerfile.yml @@ -0,0 +1,25 @@ +# Workflow to check whether changes to master concerning the Dockerfile fulfill all requirements. +name: Status checks (Dockerfile) + +on: + # Delivery pipeline already runs this on push to master. + pull_request: + paths: + - Dockerfile + +# Building the Dockerfile includes downloading the IRMA schemes. +# Therefore, we only run one check at the time, and we put a limit on the event types triggering this job. +concurrency: + group: dockerfile + +jobs: + docker-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Build Dockerfile + run: docker build -t ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} . + + - name: Test Docker image + run: docker run ghcr.io/${{ github.repository_owner }}/irma:${{ github.sha }} version From 2e0ef5caadb6dd42084e792feba5c14a0ca7f1de Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Wed, 26 Jul 2023 12:55:02 +0200 Subject: [PATCH 09/10] Chore: update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 946a1791..7825e502 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - E-mail address revalidation, addressing issues where user's e-mail addresses can be (temporary) invalid +- Publish the Docker image of the `irma`` CLI tool on ghcr.io/privacybydesign/irma ### Changed - Use separate application user in Dockerfile for entrypoint - Rename RevocationStorage's UpdateLatest function to LatestUpdates. This name better fits its behaviour. The functionality stays the same. - Validate revocation witness before revocation update is applied - RevocationStorage's EnableRevocation function does not return an error anymore if it has been enabled already +- Use a scratch Docker image as base for the Dockerfile As part of e-mail address revalidation: - `VerifyMXRecord` incorporates a check to see if there is an active network connection From 6a5089b5a73454ed36fcaf9d1bb212e932468297 Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Thu, 10 Aug 2023 11:03:15 +0200 Subject: [PATCH 10/10] Docs: improve texts in CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 753e6247..20b6f88d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,14 +8,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - E-mail address revalidation, addressing issues where user's e-mail addresses can be (temporary) invalid -- Publish the Docker image of the `irma`` CLI tool on ghcr.io/privacybydesign/irma +- Publish the Docker image of the `irma` CLI tool on ghcr.io/privacybydesign/irma ### Changed - Use separate application user in Dockerfile for entrypoint - Rename RevocationStorage's UpdateLatest function to LatestUpdates. This name better fits its behaviour. The functionality stays the same. - Validate revocation witness before revocation update is applied - RevocationStorage's EnableRevocation function does not return an error anymore if it has been enabled already -- Use a scratch Docker image as base for the Dockerfile +- Use a Docker image created from scratch as base for the Dockerfile - Custom WrapErrorPrefix function that respects the error's type - Log info message of irma.SessionError errors