From f8a93a215dca1b596714d98477f223c68676a90a Mon Sep 17 00:00:00 2001 From: marco Date: Sat, 27 Jan 2024 23:23:24 +0100 Subject: [PATCH 1/2] Decouple docker version from package release - entry point fixes for 1.6.0 - correctly override BUILD_VERSION argument - manual release workflow --- .github/workflows/publish-docker-master.yml | 46 +++++++++ .github/workflows/publish-docker-release.yml | 57 +++++++++++ .github/workflows/publish-docker.yml | 98 +++++++++++++++++++ ...ackage.yml => publish-tarball-release.yml} | 0 .../publish_docker-image_on_master-debian.yml | 71 -------------- .../publish_docker-image_on_master.yml | 71 -------------- .../release_publish_docker-image-debian.yml | 61 ------------ .../release_publish_docker-image.yml | 86 ---------------- Dockerfile | 5 +- Dockerfile.debian | 5 +- docker/docker_start.sh | 8 +- 11 files changed, 210 insertions(+), 298 deletions(-) create mode 100644 .github/workflows/publish-docker-master.yml create mode 100644 .github/workflows/publish-docker-release.yml create mode 100644 .github/workflows/publish-docker.yml rename .github/workflows/{release_publish-package.yml => publish-tarball-release.yml} (100%) delete mode 100644 .github/workflows/publish_docker-image_on_master-debian.yml delete mode 100644 .github/workflows/publish_docker-image_on_master.yml delete mode 100644 .github/workflows/release_publish_docker-image-debian.yml delete mode 100644 .github/workflows/release_publish_docker-image.yml diff --git a/.github/workflows/publish-docker-master.yml b/.github/workflows/publish-docker-master.yml new file mode 100644 index 00000000000..6f8199646c9 --- /dev/null +++ b/.github/workflows/publish-docker-master.yml @@ -0,0 +1,46 @@ +name: Publish Docker image on Push to Master + +on: + push: + branches: [ master ] + paths: + - 'pkg/**' + - 'cmd/**' + - 'plugins/**' + - 'docker/docker_start.sh' + - 'docker/config.yaml' + - '.github/workflows/publish_docker-image_on_master.yml' + - 'Dockerfile' + - 'Dockerfile.debian' + - 'go.mod' + - 'go.sum' + - 'Makefile' + +jobs: + dev-alpine: + uses: ./.github/workflows/publish-docker.yml + with: + platform: linux/amd64 + crowdsec_version: "" + image_version: dev + latest: false + push: true + slim: false + debian: false + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + dev-debian: + uses: ./.github/workflows/publish-docker.yml + with: + platform: linux/amd64 + crowdsec_version: "" + image_version: dev + latest: false + push: true + slim: false + debian: true + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} diff --git a/.github/workflows/publish-docker-release.yml b/.github/workflows/publish-docker-release.yml new file mode 100644 index 00000000000..185dcf181b3 --- /dev/null +++ b/.github/workflows/publish-docker-release.yml @@ -0,0 +1,57 @@ +name: Publish Docker images + +on: + workflow_dispatch: + inputs: + image_version: + description: Docker Image Version (base tag) + required: true + crowdsec_version: + description: Crowdsec Version (BUILD_VERSION) + required: true + latest: + description: Overwrite latest (and slim) tags? + default: false + required: true + push: + description: Really push? + default: false + required: true + +jobs: + alpine: + strategy: + matrix: + platform: ["linux/amd64", "linux/386", "linux/arm64", "linux/arm/v7", "linux/arm/v6"] + slim: [false, true] + + uses: ./.github/workflows/publish-docker.yml + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + with: + platform: ${{ matrix.platform }} + image_version: ${{ github.event.inputs.image_version }} + crowdsec_version: ${{ github.event.inputs.crowdsec_version }} + latest: ${{ github.event.inputs.latest == 'true' }} + push: ${{ github.event.inputs.push == 'true' }} + slim: ${{ matrix.slim }} + debian: false + + debian: + strategy: + matrix: + platform: ["linux/amd64", "linux/386", "linux/arm64"] + + uses: ./.github/workflows/publish-docker.yml + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + with: + platform: ${{ matrix.platform }} + image_version: ${{ github.event.inputs.image_version }} + crowdsec_version: ${{ github.event.inputs.crowdsec_version }} + latest: ${{ github.event.inputs.latest == 'true' }} + push: ${{ github.event.inputs.push == 'true' }} + slim: false + debian: true diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml new file mode 100644 index 00000000000..e475ef5cdb3 --- /dev/null +++ b/.github/workflows/publish-docker.yml @@ -0,0 +1,98 @@ +name: Publish Docker image / platform + +on: + workflow_call: + secrets: + DOCKER_USERNAME: + required: true + DOCKER_PASSWORD: + required: true + inputs: + platform: + required: true + type: string + image_version: + required: true + type: string + crowdsec_version: + required: true + type: string + latest: + required: true + type: boolean + push: + required: true + type: boolean + slim: + required: true + type: boolean + debian: + required: true + type: boolean + +jobs: + push_to_registry: + name: Push Docker image to registries + runs-on: ubuntu-latest + steps: + + - name: Check out the repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Prepare + id: prep + run: | + DOCKERHUB_IMAGE=${{ secrets.DOCKER_USERNAME }}/crowdsec + GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec + VERSION=${{ inputs.image_version }} + SLIM=${{ inputs.slim && '-slim' || '' }} + DEBIAN=${{ inputs.debian && '-debian' || '' }} + TAGS="${DOCKERHUB_IMAGE}:${VERSION}${SLIM}${DEBIAN},${GHCR_IMAGE}:${VERSION}${SLIM}${DEBIAN}" + if [[ ${{ inputs.latest }} == true ]]; then + if [[ ${{ inputs.slim }} == true ]]; then + TAGS=$TAGS,${DOCKERHUB_IMAGE}:slim${DEBIAN},${GHCR_IMAGE}:slim${DEBIAN} + else + TAGS=$TAGS,${DOCKERHUB_IMAGE}:latest${DEBIAN},${GHCR_IMAGE}:latest${DEBIAN} + fi + fi + echo "tags=${TAGS}" >> $GITHUB_OUTPUT + echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + config: .github/buildkit.toml + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push image + uses: docker/build-push-action@v4 + with: + context: . + file: ./Dockerfile${{ inputs.debian && '.debian' || '' }} + push: ${{ inputs.push }} + tags: ${{ steps.prep.outputs.tags }} + target: ${{ inputs.slim && 'slim' || 'full' }} + platforms: ${{ inputs.platform }} + labels: | + org.opencontainers.image.source=${{ github.event.repository.html_url }} + org.opencontainers.image.created=${{ steps.prep.outputs.created }} + org.opencontainers.image.revision=${{ github.sha }} + build-args: | + BUILD_VERSION=${{ inputs.crowdsec_version }} diff --git a/.github/workflows/release_publish-package.yml b/.github/workflows/publish-tarball-release.yml similarity index 100% rename from .github/workflows/release_publish-package.yml rename to .github/workflows/publish-tarball-release.yml diff --git a/.github/workflows/publish_docker-image_on_master-debian.yml b/.github/workflows/publish_docker-image_on_master-debian.yml deleted file mode 100644 index 17332adf0f3..00000000000 --- a/.github/workflows/publish_docker-image_on_master-debian.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Publish Debian Docker image on Push to Master - -on: - push: - branches: [ master ] - paths: - - 'pkg/**' - - 'cmd/**' - - 'plugins/**' - - 'docker/docker_start.sh' - - 'docker/config.yaml' - - '.github/workflows/publish_docker-image_on_master-debian.yml' - - 'Dockerfile.debian' - - 'go.mod' - - 'go.sum' - - 'Makefile' - -jobs: - push_to_registry: - name: Push Debian Docker image to Docker Hub - runs-on: ubuntu-latest - if: ${{ github.repository_owner == 'crowdsecurity' }} - steps: - - - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=crowdsecurity/crowdsec - GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec - VERSION=dev-debian - TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}" - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - config: .github/buildkit.toml - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push full image - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile.debian - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags }} - platforms: linux/amd64 - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=min diff --git a/.github/workflows/publish_docker-image_on_master.yml b/.github/workflows/publish_docker-image_on_master.yml deleted file mode 100644 index 34529020012..00000000000 --- a/.github/workflows/publish_docker-image_on_master.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Publish Docker image on Push to Master - -on: - push: - branches: [ master ] - paths: - - 'pkg/**' - - 'cmd/**' - - 'plugins/**' - - 'docker/docker_start.sh' - - 'docker/config.yaml' - - '.github/workflows/publish_docker-image_on_master.yml' - - 'Dockerfile' - - 'go.mod' - - 'go.sum' - - 'Makefile' - -jobs: - push_to_registry: - name: Push Docker image to Docker Hub - runs-on: ubuntu-latest - if: ${{ github.repository_owner == 'crowdsecurity' }} - steps: - - - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=crowdsecurity/crowdsec - GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec - VERSION=dev - TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}" - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - config: .github/buildkit.toml - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push full image - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags }} - platforms: linux/amd64 - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=min diff --git a/.github/workflows/release_publish_docker-image-debian.yml b/.github/workflows/release_publish_docker-image-debian.yml deleted file mode 100644 index e766dae0966..00000000000 --- a/.github/workflows/release_publish_docker-image-debian.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Publish Docker Debian image - -on: - release: - types: - - released - - prereleased - workflow_dispatch: - -jobs: - push_to_registry: - name: Push Docker debian image to Docker Hub - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=crowdsecurity/crowdsec - VERSION=bullseye - if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g') - elif [[ $GITHUB_REF == refs/pull/* ]]; then - VERSION=pr-${{ github.event.number }} - fi - TAGS="${DOCKER_IMAGE}:${VERSION}-debian" - if [[ "${{ github.event.action }}" == "released" ]]; then - TAGS=$TAGS,${DOCKER_IMAGE}:latest-debian - fi - echo "version=${VERSION}" >> $GITHUB_OUTPUT - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - config: .github/buildkit.toml - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile.debian - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags }} - platforms: linux/amd64,linux/arm64,linux/386 - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} diff --git a/.github/workflows/release_publish_docker-image.yml b/.github/workflows/release_publish_docker-image.yml deleted file mode 100644 index fa4eb8996a4..00000000000 --- a/.github/workflows/release_publish_docker-image.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Publish Docker image - -on: - release: - types: - - released - - prereleased - -jobs: - push_to_registry: - name: Push Docker image to Docker Hub - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=crowdsecurity/crowdsec - GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec - VERSION=edge - if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g') - elif [[ $GITHUB_REF == refs/pull/* ]]; then - VERSION=pr-${{ github.event.number }} - fi - TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}" - TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim,${GHCR_IMAGE}:${VERSION}-slim" - if [[ ${{ github.event.action }} == released ]]; then - TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest - TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim,${GHCR_IMAGE}:slim - fi - echo "version=${VERSION}" >> $GITHUB_OUTPUT - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - echo "tags_slim=${TAGS_SLIM}" >> $GITHUB_OUTPUT - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - config: .github/buildkit.toml - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push slim image - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags_slim }} - target: slim - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386 - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} - - - name: Build and push full image - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags }} - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386 - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} diff --git a/Dockerfile b/Dockerfile index 7470beb57a2..0409f6e7c52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,7 @@ # vim: set ft=dockerfile: -ARG GOVERSION=1.21.6 -ARG BUILD_VERSION +FROM golang:1.21.6-alpine3.18 AS build -FROM golang:${GOVERSION}-alpine3.18 AS build +ARG BUILD_VERSION WORKDIR /go/src/crowdsec diff --git a/Dockerfile.debian b/Dockerfile.debian index bc5b0aa2d2e..ef9d905ac22 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -1,8 +1,7 @@ # vim: set ft=dockerfile: -ARG GOVERSION=1.21.6 -ARG BUILD_VERSION +FROM golang:1.21.6-bookworm AS build -FROM golang:${GOVERSION}-bookworm AS build +ARG BUILD_VERSION WORKDIR /go/src/crowdsec diff --git a/docker/docker_start.sh b/docker/docker_start.sh index 1e44b64ac1c..5393feeb87d 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -3,7 +3,7 @@ # shellcheck disable=SC2292 # allow [ test ] syntax # shellcheck disable=SC2310 # allow "if function..." syntax with -e -# set -e +set -e shopt -s inherit_errexit # match true, TRUE, True, tRuE, etc. @@ -109,6 +109,8 @@ cscli_if_clean() { for obj in $objs; do if cscli "$itemtype" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then echo "Object $itemtype/$obj is tainted, skipping" + elif cscli "$itemtype" inspect "$obj" -o json | yq -e '.local // false' >/dev/null 2>&1; then + echo "Object $itemtype/$obj is local, skipping" else # # Too verbose? Only show errors if not in debug mode # if [ "$DEBUG" != "true" ]; then @@ -301,8 +303,8 @@ fi conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)' ## Install hub items -cscli hub update -cscli hub upgrade +cscli hub update || true +cscli hub upgrade || true cscli_if_clean parsers install crowdsecurity/docker-logs cscli_if_clean parsers install crowdsecurity/cri-logs From 5c90b9f7abc9c66b0bdc98d423e273dcdbec992f Mon Sep 17 00:00:00 2001 From: marco Date: Mon, 29 Jan 2024 17:28:13 +0100 Subject: [PATCH 2/2] fixed paths --- .github/workflows/publish-docker-master.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-docker-master.yml b/.github/workflows/publish-docker-master.yml index 6f8199646c9..e6f9cebf75d 100644 --- a/.github/workflows/publish-docker-master.yml +++ b/.github/workflows/publish-docker-master.yml @@ -9,7 +9,8 @@ on: - 'plugins/**' - 'docker/docker_start.sh' - 'docker/config.yaml' - - '.github/workflows/publish_docker-image_on_master.yml' + - '.github/workflows/publish_docker-master.yml' + - '.github/workflows/publish-docker.yml' - 'Dockerfile' - 'Dockerfile.debian' - 'go.mod'