docker build flavors: slim, with-plugins, with-geoip, full (#1862)

.github/workflows/release_publish_docker-image.yml

@@ -29,11 +29,14 @@ jobs:
             VERSION=pr-${{ github.event.number }}
           fi
           TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
+          TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim"
           if [[ ${{ github.event.action }} == released ]]; then
             TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
+            TAGS_SLIM=$TAGS,${DOCKER_IMAGE}:slim
           fi
           echo ::set-output name=version::${VERSION}
           echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=tags_slim::${TAGS_SLIM}
           echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
       -
         name: Set up QEMU
@@ -54,8 +57,25 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
+
+      -
+        name: Build and push slim image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.prep.outputs.tags_slim }}
+          build-args: |
+            BUILD_ENV=slim
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
+          labels: |
+            org.opencontainers.image.source=${{ github.event.repository.html_url }}
+            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}
+
       -
-        name: Build and push
+        name: Build and push full image
         uses: docker/build-push-action@v2
         with:
           context: .

Dockerfile

@@ -1,33 +1,51 @@
+ARG BUILD_ENV=full
 ARG GOVERSION=1.19
 
 FROM golang:${GOVERSION}-alpine AS build
 
 WORKDIR /go/src/crowdsec
 
+COPY . .
+
 # wizard.sh requires GNU coreutils
-RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils
+RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \
+    SYSTEM="docker" make release && \
+    cd crowdsec-v* && \
+    ./wizard.sh --docker-mode && \
+    cd - && \
+    cscli hub update && \
+    cscli collections install crowdsecurity/linux && \
+    cscli parsers install crowdsecurity/whitelists
 
-COPY . .
+FROM alpine:latest as build-slim
 
-RUN SYSTEM="docker" make release
-RUN cd crowdsec-v* && ./wizard.sh --docker-mode && cd -
-RUN cscli hub update && cscli collections install crowdsecurity/linux && cscli parsers install crowdsecurity/whitelists
-FROM alpine:latest
 RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata yq bash && \
     mkdir -p /staging/etc/crowdsec && \
-    mkdir -p /staging/var/lib/crowdsec
+    mkdir -p /staging/var/lib/crowdsec && \
+    mkdir -p /var/lib/crowdsec/data
 COPY --from=build /etc/crowdsec /staging/etc/crowdsec
-COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
 COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
 COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
 COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
 COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
-#Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
-#The files are here for reference, as users will need to mount a new version to be actually able to use notifications
+
+ENTRYPOINT /bin/bash docker_start.sh
+
+FROM build-slim as build-plugins
+# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
+# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
 COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
 COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
 COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
 COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
 COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
 
-ENTRYPOINT /bin/bash docker_start.sh
+FROM build-slim as build-geoip
+
+COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
+
+FROM build-plugins as build-full
+
+COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
+
+FROM build-${BUILD_ENV}

Dockerfile.debian

@@ -1,46 +1,64 @@
+ARG BUILD_ENV=full
 ARG GOVERSION=1.19
 
 FROM golang:${GOVERSION}-bullseye AS build
 
 WORKDIR /go/src/crowdsec
 
-# wizard.sh requires GNU coreutils
-RUN apt-get update && apt-get install -y git gcc libc-dev make bash gettext binutils-gold coreutils tzdata python3 python3-pip
-
 COPY . .
 
-RUN SYSTEM="docker" make release
-RUN cd crowdsec-v* && ./wizard.sh --docker-mode && cd -
-RUN cscli hub update && cscli collections install crowdsecurity/linux && cscli parsers install crowdsecurity/whitelists
-RUN GO111MODULE=on go get github.com/mikefarah/yq/v4
+# wizard.sh requires GNU coreutils
+RUN apt-get update && \
+    apt-get install -y git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
+    SYSTEM="docker" make release && \
+    cd crowdsec-v* && \
+    ./wizard.sh --docker-mode && \
+    cd - && \
+    cscli hub update && \
+    cscli collections install crowdsecurity/linux && \
+    cscli parsers install crowdsecurity/whitelists && \
+    go install github.com/mikefarah/yq/v4@latest
 
-FROM debian:bullseye-slim
+FROM debian:bullseye-slim as build-slim
 
-RUN apt-get update
-RUN apt-get install -y -q --install-recommends --no-install-suggests \
+RUN apt-get update && \
+    apt-get install -y -q --install-recommends --no-install-suggests \
     procps \
     systemd \
     iproute2 \
     ca-certificates \
     bash \
     tzdata && \
     mkdir -p /staging/etc/crowdsec && \
-    mkdir -p /staging/var/lib/crowdsec
+    mkdir -p /staging/var/lib/crowdsec && \
+    mkdir -p /var/lib/crowdsec/data
 
 COPY --from=build /go/bin/yq /usr/local/bin/yq
 COPY --from=build /etc/crowdsec /staging/etc/crowdsec
-COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
 COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
 COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
 COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
 COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
 RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml
-#Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
-#The files are here for reference, as users will need to mount a new version to be actually able to use notifications
+
+ENTRYPOINT /bin/bash docker_start.sh
+
+FROM build-slim as build-plugins
+
+# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
+# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
 COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
 COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
 COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
 COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
 COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
 
-ENTRYPOINT /bin/bash docker_start.sh
+FROM build-slim as build-geoip
+
+COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
+
+FROM build-plugins as build-full
+
+COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
+
+FROM build-${BUILD_ENV}

docker/README.md

@@ -10,13 +10,53 @@ Crowdsec - An open-source, lightweight agent to detect and respond to bad behavi
 
 # How to use this image
 
-## Docker images available
-crowdsec will use Alpine as default container. A debian container is also available with systemd for journalctl support. Simply add `-debian` to your tag to use this. Please be aware that debian containers are not available on all version, since the feature was implemented after the release of version 1.3.0
+## Docker image versions
+
+All the following versions are available on Docker Hub for 386, amd64, arm/v6, arm/v7, arm64.
+
+### Alpine
+
+ - crowdsecurity/crowdsec:{version}
+
+Recommended for production usage. Also available on GitHub (ghrc.io).
+
+ - crowdsecurity/crowdsec:latest
+
+For development and testing.
+
+since v1.4.2:
+
+ - crowdsecurity/crowdsec:slim
+
+Reduced size by 60%, does not include notifier plugins nor the GeoIP database.
+If you need these details on decisions, running `cscli hub upgrade` inside the
+container downloads the GeoIP database at runtime.
+
+
+### Debian (since v1.3.3)
+
+ - crowdsecurity/crowdsec:{version}-debian
+ - crowdsecurity/crowdsec:latest-debian
+
+The debian version includes support for systemd and journalctl.
+
+### Custom
+
+You can build your own images with Dockerfile and Dockerfile-debian.
+
+For example, if you want a Debian version without plugin notifiers:
+
+```console
+$ docker build -f Dockerfile.debian --build-arg=BUILD_ENV=slim
+```
+
+supported values for BUILD_ENV are: full, with-geoip, with-plugins, slim.
+
 
 ## Required configuration
 
 ### Journalctl (only for debian image)
-To use journalctl (only for debian image) as log stream, eventually from the `DSN` environment variable, it's important that you mount the journal log from the host to the container it self.
+To use journalctl (only with the debian image) as a log stream, eventually from the `DSN` environment variable, it's important to mount the journal log from the host to the container itself.
 This can be done by adding the following volume mount to your docker command:
 
 ```
@@ -77,28 +117,28 @@ docker run -d \
 Check this full stack example using docker-compose: https://github.com/crowdsecurity/example-docker-compose
 # How to extend this image
 ## Full configuration
-The container is built with specific docker [configuration](https://github.com/crowdsecurity/crowdsec/blob/master/docker/config.yaml). If you need to change it, bind `/etc/crowdsec/config.yaml` to your local configuration file
+The container is built with a specific docker [configuration](https://github.com/crowdsecurity/crowdsec/blob/master/docker/config.yaml). If you need to change it, bind `/etc/crowdsec/config.yaml` to your local configuration file
 ## Notifications
 If you wish to use the [notification system](https://docs.crowdsec.net/docs/notification_plugins/intro), you will need to mount at least a custom `profiles.yaml` and a notification configuration to `/etc/crowdsec/notifications`
 
 # Deployment use cases
-Crowdsec is composed of an `agent` that parse logs and creates `alerts` that `local API` or `LAPI` transform into decisions. Both can run in the same process but also on separated containers as it makes sense in complex configurations to have agents on the same machines as the protected component and a LAPI that gather all signals from agents and communicate with the `central api`.
+Crowdsec is composed of an `agent` that parses logs and creates `alerts` that `local API` or `LAPI` transform into decisions. Both can run in the same process but also on separated containers as it makes sense in complex configurations to have agents on the same machines as the protected component and a LAPI that gather all signals from agents and communicate with the `central api`.
 
 ## Register a new agent with LAPI
 ```shell
 docker exec -it crowdsec_lapi_container_name cscli machines add agent_user_name --password agent_password
 ```
 
 ## Run an agent connected to LAPI
-Add following environment variables to your docker run command:
+Add the following environment variables to your docker run command:
 * `DISABLE_LOCAL_API=true`
 * `AGENT_USERNAME="agent_user_name"` - agent_user_name previously registered with LAPI
 * `AGENT_PASSWORD="agent_password"` - agent_password previously registered with LAPI
 * `LOCAL_API_URL="http://LAPI_host:LAPI_port"`
 
 # Next steps
 ## Bouncers
-Crowdsec being a detection component, remediation is implemented using `bouncers`. Each bouncer protect a specific component. Find out more:
+Crowdsec being a detection component, remediation is implemented using `bouncers`. Each bouncer protects a specific component. Find out more:
 
 https://hub.crowdsec.net/browse/#bouncers
 
@@ -110,17 +150,17 @@ You can automatically register bouncers with the crowdsec container on startup u
 
 To use environment variables, they should be in the format `BOUNCER_KEY_<name>=<key>`. e.g. `BOUNCER_KEY_nginx=mysecretkey12345`.
 
-To use Docker secrets, the secret should be named `bouncer_key_<name>` with a content of `<key>`. e.g. `bouncer_key_nginx` with a content of `mysecretkey12345`.
+To use Docker secrets, the secret should be named `bouncer_key_<name>` with a content of `<key>`. e.g. `bouncer_key_nginx` with content `mysecretkey12345`.
 
 A bouncer key can be any string but we recommend an alphanumeric value to keep consistent with crowdsec-generated keys and avoid problems with escaping special characters.
 
 ## Console
-We provide a web based interface to get more from Crowdsec: https://docs.crowdsec.net/docs/console
+We provide a web-based interface to get more from Crowdsec: https://docs.crowdsec.net/docs/console
 
 Subscribe here: https://app.crowdsec.net
 
 # Caveats
-Using binds rather than named volumes ([more explanation here](https://docs.docker.com/storage/volumes/)) results in more complexity as you'll have to bind relevant files one by one where with named volumes you can mount full configuration and data folders. On the other hand, named volumes are less straightforward to navigate.
+Using binds rather than named volumes ([more explanation here](https://docs.docker.com/storage/volumes/)) results in more complexity as you'll have to bind relevant files one by one whereas with named volumes you can mount full configuration and data folders. On the other hand, named volumes are less straightforward to navigate.
 
 # Reference
 ## Environment Variables