From 42f6a915938764a24e4b582c239b64c22f46c848 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 22 Jan 2024 23:05:42 -0700 Subject: [PATCH 1/9] Try speeding up QEMU builds --- .github/workflows/docker.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 38c9f7d6..9e5f031f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,7 +11,7 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build: + build-linux: runs-on: ubuntu-latest permissions: contents: read @@ -21,11 +21,11 @@ jobs: matrix: platform: - linux/amd64 - - linux/arm/v6 - - linux/arm/v7 - - linux/arm64 +# - linux/arm/v6 +# - linux/arm/v7 +# - linux/arm64 - steps: + steps: &dockerBuild - name: Prepare run: | platform=${{ matrix.platform }} @@ -76,9 +76,25 @@ jobs: if-no-files-found: error retention-days: 1 + build-macos: + runs-on: macos-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + platform: +# - linux/amd64 + - linux/arm/v6 + - linux/arm/v7 + - linux/arm64 + + steps: *dockerBuild + merge: runs-on: ubuntu-latest - needs: [build] + needs: [build-linux, build-macos] steps: - name: Download digests uses: actions/download-artifact@v4 From 95fc0f8f6cdf6c45bb58d8bc1759a8bf18c44859 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 22 Jan 2024 23:06:35 -0700 Subject: [PATCH 2/9] Add branch for now --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9e5f031f..ace2af2d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,7 +2,7 @@ name: Create and publish a Docker image on: push: - branches: ['main'] + branches: ['main', 'travis/qemu-speedup'] release: types: ['published'] From 046d2d8a37f7cab06dc5993e6c38a4accf10fa32 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 22 Jan 2024 23:07:42 -0700 Subject: [PATCH 3/9] Fine, don't use anchors --- .github/workflows/docker.yml | 57 ++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ace2af2d..6ec4aeb1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,11 +21,8 @@ jobs: matrix: platform: - linux/amd64 -# - linux/arm/v6 -# - linux/arm/v7 -# - linux/arm64 - steps: &dockerBuild + steps: - name: Prepare run: | platform=${{ matrix.platform }} @@ -85,12 +82,60 @@ jobs: fail-fast: false matrix: platform: -# - linux/amd64 - linux/arm/v6 - linux/arm/v7 - linux/arm64 - steps: *dockerBuild + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Checkout + uses: actions/checkout@v4 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v5 + with: + context: . + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 merge: runs-on: ubuntu-latest From 2e5a53e60eff936af1c19412f6b46dddae82d66c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 22 Jan 2024 23:14:48 -0700 Subject: [PATCH 4/9] Adjust matrix values --- .github/workflows/docker.yml | 86 +++++++++--------------------------- 1 file changed, 20 insertions(+), 66 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6ec4aeb1..af8e1ed0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,16 +11,24 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-linux: - runs-on: ubuntu-latest + build: + runs-on: ${{ matrix.os }} permissions: contents: read packages: write strategy: fail-fast: false matrix: - platform: - - linux/amd64 + os: [ubuntu-latest, macos-latest] + include: + - os: ubuntu-latest + platform: + - linux/amd64 + - os: macos-latest + platform: + - linux/arm/v6 + - linux/arm/v7 + - linux/arm64 steps: - name: Prepare @@ -28,69 +36,15 @@ jobs: platform=${{ matrix.platform }} echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - name: Checkout - uses: actions/checkout@v4 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push by digest - id: build - uses: docker/build-push-action@v5 - with: - context: . - platforms: ${{ matrix.platform }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - - - name: Export digest + - name: Setup docker (missing on MacOS) + if: runner.os == 'macos' run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - build-macos: - runs-on: macos-latest - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - platform: - - linux/arm/v6 - - linux/arm/v7 - - linux/arm64 + brew install docker + colima start - steps: - - name: Prepare - run: | - platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + # For testcontainers to find the Colima socket + # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running + sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock - name: Checkout uses: actions/checkout@v4 @@ -139,7 +93,7 @@ jobs: merge: runs-on: ubuntu-latest - needs: [build-linux, build-macos] + needs: [build] steps: - name: Download digests uses: actions/download-artifact@v4 From 55fc922465bc3d8788c06e65fbc1a77a7bfbf9ab Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 22 Jan 2024 23:15:52 -0700 Subject: [PATCH 5/9] Fix syntax --- .github/workflows/docker.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index af8e1ed0..dad603cf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,13 +22,13 @@ jobs: os: [ubuntu-latest, macos-latest] include: - os: ubuntu-latest - platform: - - linux/amd64 + platform: linux/amd64 - os: macos-latest - platform: - - linux/arm/v6 - - linux/arm/v7 - - linux/arm64 + platform: linux/arm/v6 + - os: macos-latest + platform: linux/arm/v7 + - os: macos-latest + platform: linux/arm64 steps: - name: Prepare From ca8ff663203383bd6bf48cbd932fbfec29953f82 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 22 Jan 2024 23:21:52 -0700 Subject: [PATCH 6/9] Adjust config --- .github/workflows/docker.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dad603cf..9fee4dcb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -20,15 +20,20 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - include: + platform: + - linux/amd64 + - linux/arm/v6 + - linux/arm/v7 + - linux/arm64 + exclude: - os: ubuntu-latest - platform: linux/amd64 - - os: macos-latest platform: linux/arm/v6 - - os: macos-latest + - os: ubuntu-latest platform: linux/arm/v7 - - os: macos-latest + - os: ubuntu-latest platform: linux/arm64 + - os: macos-latest + platform: linux/amd64 steps: - name: Prepare From 5c3f854a17fb63a14cefc79ca439b1ed2448d9db Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 22 Jan 2024 23:26:23 -0700 Subject: [PATCH 7/9] Use the M1 fleet --- .github/workflows/docker.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9fee4dcb..9e8ee1fb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest-xlarge] platform: - linux/amd64 - linux/arm/v6 @@ -32,7 +32,7 @@ jobs: platform: linux/arm/v7 - os: ubuntu-latest platform: linux/arm64 - - os: macos-latest + - os: macos-latest-xlarge platform: linux/amd64 steps: @@ -41,15 +41,15 @@ jobs: platform=${{ matrix.platform }} echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - name: Setup docker (missing on MacOS) - if: runner.os == 'macos' - run: | - brew install docker - colima start - - # For testcontainers to find the Colima socket - # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running - sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock +# - name: Setup docker (missing on MacOS) +# if: runner.os == 'macos' +# run: | +# brew install docker +# colima start +# +# # For testcontainers to find the Colima socket +# # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running +# sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock - name: Checkout uses: actions/checkout@v4 From 2ea17e5cc578c4c4002dd7bdd4a9f7924d2f3d33 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 22 Jan 2024 23:30:54 -0700 Subject: [PATCH 8/9] Add concurrency and re-install docker --- .github/workflows/docker.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9e8ee1fb..77498f2f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,6 +10,10 @@ env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + jobs: build: runs-on: ${{ matrix.os }} @@ -41,15 +45,15 @@ jobs: platform=${{ matrix.platform }} echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV -# - name: Setup docker (missing on MacOS) -# if: runner.os == 'macos' -# run: | -# brew install docker -# colima start -# -# # For testcontainers to find the Colima socket -# # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running -# sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock + - name: Setup docker (missing on MacOS) + if: runner.os == 'macos' + run: | + brew install docker + colima start + + # For testcontainers to find the Colima socket + # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running + sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock - name: Checkout uses: actions/checkout@v4 From 3b6c3e878a9be6047d086c130b13c140d84ff436 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 22 Jan 2024 23:31:50 -0700 Subject: [PATCH 9/9] Fix docker install? --- .github/workflows/docker.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 77498f2f..03954efa 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -49,11 +49,6 @@ jobs: if: runner.os == 'macos' run: | brew install docker - colima start - - # For testcontainers to find the Colima socket - # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running - sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock - name: Checkout uses: actions/checkout@v4