From cc15fe36def51e3be37009dffb95ee877ed2ba8a Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Thu, 12 Sep 2024 17:16:28 -0700 Subject: [PATCH 1/3] Fix test setup to run on ARM instances --- Dockerfile.build | 3 ++- Makefile | 11 ++++++++--- dev-tools/e2e/docker-compose.yml | 4 ++-- dev-tools/integration/docker-compose.yml | 10 ++++------ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Dockerfile.build b/Dockerfile.build index ea92b5082..8f0fec22a 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -1,5 +1,6 @@ ARG GO_VERSION -FROM docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian11 +ARG ARCH=main # ARCH should be main or arm +FROM docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-${ARCH} RUN \ apt-get update \ diff --git a/Makefile b/Makefile index b944cb30a..21a19d5c9 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,11 @@ endif DOCKER_IMAGE_TAG?=${VERSION} DOCKER_IMAGE?=docker.elastic.co/fleet-server/fleet-server +ifeq ($(shell uname -p),arm) +DOCKERARCH=arm +else +DOCKERARCH=main +endif PLATFORM_TARGETS=$(addprefix release-, $(PLATFORMS)) COVER_TARGETS=$(addprefix cover-, $(PLATFORMS)) @@ -248,7 +253,7 @@ else endif build-releaser: ## - Build a Docker image to run make package including all build tools - docker build -t $(BUILDER_IMAGE) -f Dockerfile.build --build-arg GO_VERSION=$(GO_VERSION) . + docker build -t $(BUILDER_IMAGE) -f Dockerfile.build --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=${DOCKERARCH} . .PHONY: docker-release docker-release: build-releaser ## - Builds a release for all platforms in a dockerised environment @@ -257,7 +262,7 @@ docker-release: build-releaser ## - Builds a release for all platforms in a dock .PHONY: docker-cover-e2e-binaries docker-cover-e2e-binaries: build-releaser ## Build for local architecture and for linux/amd64 for docker images. - docker run --rm -u $(shell id -u):$(shell id -g) --volume $(PWD):/go/src/github.com/elastic/fleet-server -e SNAPSHOT=true $(BUILDER_IMAGE) cover-linux/amd64 cover-$(shell go env GOOS)/$(shell go env GOARCH) + docker run --rm -u $(shell id -u):$(shell id -g) --volume $(PWD):/go/src/github.com/elastic/fleet-server -e SNAPSHOT=true $(BUILDER_IMAGE) cover-linux/$(shell go env GOARCH) cover-$(shell go env GOOS)/$(shell go env GOARCH) .PHONY: release release: $(PLATFORM_TARGETS) ## - Builds a release. Specify exact platform with PLATFORMS env. @@ -336,7 +341,7 @@ test-int-set: ## - Run integration tests without setup .PHONY: build-e2e-agent-image build-e2e-agent-image: docker-cover-e2e-binaries ## - Build a custom elastic-agent image with fleet-server binaries with coverage enabled injected @printf "${CMD_COLOR_ON} Creating test e2e agent image\n${CMD_COLOR_OFF}" - GOARCH=amd64 ./dev-tools/e2e/build.sh + ./dev-tools/e2e/build.sh .PHONY: e2e-certs e2e-certs: ## - Use openssl to create a CA, encrypted private key, and signed fleet-server cert testing purposes diff --git a/dev-tools/e2e/docker-compose.yml b/dev-tools/e2e/docker-compose.yml index 11c43eb5d..c260d2eee 100644 --- a/dev-tools/e2e/docker-compose.yml +++ b/dev-tools/e2e/docker-compose.yml @@ -1,6 +1,6 @@ services: kibana: - image: "docker.elastic.co/kibana/kibana:${ELASTICSEARCH_VERSION}-amd64" + image: "docker.elastic.co/kibana/kibana:${ELASTICSEARCH_VERSION}" container_name: kibana healthcheck: test: ["CMD", "curl", "-f", "${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@localhost:5601/api/status"] @@ -17,7 +17,7 @@ services: networks: - integration apm-server: - image: "docker.elastic.co/apm/apm-server:${ELASTICSEARCH_VERSION}-amd64" + image: "docker.elastic.co/apm/apm-server:${ELASTICSEARCH_VERSION}" container_name: apm-server # curl is not in the apm-server image #healthcheck: diff --git a/dev-tools/integration/docker-compose.yml b/dev-tools/integration/docker-compose.yml index c5eead706..482fea5eb 100644 --- a/dev-tools/integration/docker-compose.yml +++ b/dev-tools/integration/docker-compose.yml @@ -3,7 +3,7 @@ volumes: driver: local services: setup: - image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION}-amd64 + image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION} volumes: - certs:/usr/share/elasticsearch/config/certs user: "0" @@ -45,7 +45,7 @@ services: find . -type f -exec chmod 640 \{\} \;; '; elasticsearch: - image: "docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION}-amd64" + image: "docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION}" container_name: elasticsearch environment: - node.name=es01 @@ -84,12 +84,11 @@ services: - certs:/usr/share/elasticsearch/config/certs ports: - 127.0.0.1:9200:9200 - elasticsearch-remote: depends_on: setup: - condition: service_healthy - image: "docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION}-amd64" + condition: service_completed_successfully + image: "docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION}" container_name: elasticsearch-remote environment: - node.name=es02 @@ -127,4 +126,3 @@ services: - certs:/usr/share/elasticsearch/config/certs ports: - 127.0.0.1:9201:9200 - From aa6b460e753b004fce239dace5b7ef09d089f247 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Fri, 13 Sep 2024 10:07:50 -0700 Subject: [PATCH 2/3] Change arg to TARGETARCH --- Dockerfile.build | 4 ++-- Makefile | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Dockerfile.build b/Dockerfile.build index 8f0fec22a..f95994a2e 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -1,6 +1,6 @@ ARG GO_VERSION -ARG ARCH=main # ARCH should be main or arm -FROM docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-${ARCH} +ARG TARGETARCH # Should be main or arm +FROM docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-${TARGETARCH} RUN \ apt-get update \ diff --git a/Makefile b/Makefile index 21a19d5c9..d37d52741 100644 --- a/Makefile +++ b/Makefile @@ -43,12 +43,6 @@ endif DOCKER_IMAGE_TAG?=${VERSION} DOCKER_IMAGE?=docker.elastic.co/fleet-server/fleet-server -ifeq ($(shell uname -p),arm) -DOCKERARCH=arm -else -DOCKERARCH=main -endif - PLATFORM_TARGETS=$(addprefix release-, $(PLATFORMS)) COVER_TARGETS=$(addprefix cover-, $(PLATFORMS)) COMMIT=$(shell git rev-parse --short HEAD) @@ -253,7 +247,12 @@ else endif build-releaser: ## - Build a Docker image to run make package including all build tools - docker build -t $(BUILDER_IMAGE) -f Dockerfile.build --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=${DOCKERARCH} . +ifeq ($(shell uname -p),arm) + $(eval DOCKERARCH := arm) +else + $(eval DOCKERARCH := main) +endif + docker build -t $(BUILDER_IMAGE) -f Dockerfile.build --build-arg GO_VERSION=$(GO_VERSION) --build-arg TARGETARCH=${DOCKERARCH} . .PHONY: docker-release docker-release: build-releaser ## - Builds a release for all platforms in a dockerised environment From 608a7cc417b869235d582755d154cfd32073e814 Mon Sep 17 00:00:00 2001 From: michel-laterman Date: Thu, 19 Sep 2024 12:28:19 -0700 Subject: [PATCH 3/3] Change DOCKERARCH to SUFFIX, change to main-debian11 --- Dockerfile.build | 4 ++-- Makefile | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile.build b/Dockerfile.build index f95994a2e..db7f7d24c 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -1,6 +1,6 @@ ARG GO_VERSION -ARG TARGETARCH # Should be main or arm -FROM docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-${TARGETARCH} +ARG SUFFIX # Should be main-debian11 or arm +FROM docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-${SUFFIX} RUN \ apt-get update \ diff --git a/Makefile b/Makefile index d37d52741..f41624572 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,8 @@ BUILDMODE_windows_amd64=-buildmode=pie BUILDMODE_darwin_amd64=-buildmode=pie BUILDMODE_darwin_arm64=-buildmode=pie -BUILDER_IMAGE=docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian11 +CROSSBUILD_SUFFIX=main-debian11 +BUILDER_IMAGE=docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-${CROSSBUILD_SUFFIX} #Benchmark related targets BENCH_BASE ?= benchmark-$(COMMIT).out @@ -248,11 +249,11 @@ endif build-releaser: ## - Build a Docker image to run make package including all build tools ifeq ($(shell uname -p),arm) - $(eval DOCKERARCH := arm) + $(eval SUFFIX := arm) else - $(eval DOCKERARCH := main) + $(eval SUFFIX := ${CROSSBUILD_SUFFIX}) endif - docker build -t $(BUILDER_IMAGE) -f Dockerfile.build --build-arg GO_VERSION=$(GO_VERSION) --build-arg TARGETARCH=${DOCKERARCH} . + docker build -t $(BUILDER_IMAGE) -f Dockerfile.build --build-arg GO_VERSION=$(GO_VERSION) --build-arg SUFFIX=${SUFFIX} . .PHONY: docker-release docker-release: build-releaser ## - Builds a release for all platforms in a dockerised environment