From 6a5fcfdc569bb60f6e302c467be82b8ff954f067 Mon Sep 17 00:00:00 2001 From: Qingchuan Hao Date: Fri, 13 Dec 2024 11:25:54 +0000 Subject: [PATCH 1/3] integrate binary build in windows dockerfile --- Makefile | 4 ++-- cloud-node-manager-windows-hpc.Dockerfile | 11 ++++++++++- cloud-node-manager-windows.Dockerfile | 12 ++++++++++-- cloud-node-manager.Dockerfile | 2 -- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 533e91efec..3342dd8530 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,7 @@ build-node-image-linux: buildx-setup ## Build node-manager image. --sbom=false .PHONY: build-node-image-windows -build-node-image-windows: buildx-setup $(BIN_DIR)/azure-cloud-node-manager.exe ## Build node-manager image for Windows. +build-node-image-windows: buildx-setup ## Build node-manager image for Windows. $(DOCKER_BUILDX) build --pull \ --output=type=$(OUTPUT_TYPE) \ --platform windows/$(ARCH) \ @@ -180,7 +180,7 @@ build-node-image-windows: buildx-setup $(BIN_DIR)/azure-cloud-node-manager.exe # --sbom=false .PHONY: build-node-image-windows-hpc -build-node-image-windows-hpc: buildx-setup $(BIN_DIR)/azure-cloud-node-manager.exe ## Build node-manager image for Windows. +build-node-image-windows-hpc: buildx-setup ## Build node-manager image for Windows. $(DOCKER_BUILDX) build --pull \ --output=type=$(OUTPUT_TYPE) \ --platform windows/$(ARCH) \ diff --git a/cloud-node-manager-windows-hpc.Dockerfile b/cloud-node-manager-windows-hpc.Dockerfile index 2836dc7406..686c0ea85f 100644 --- a/cloud-node-manager-windows-hpc.Dockerfile +++ b/cloud-node-manager-windows-hpc.Dockerfile @@ -14,7 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. ARG ARCH=amd64 + +# build windows cloud noder manager binary +FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS builder +ARG ENABLE_GIT_COMMAND=true +ARG ARCH +WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure +COPY . . +RUN make bin/azure-cloud-node-manager.exe ENABLE_GIT_COMMAND=${ENABLE_GIT_COMMAND} ARCH=${ARCH} + FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0 ARG ARCH -COPY bin/azure-cloud-node-manager-${ARCH}.exe /cloud-node-manager.exe +COPY --from=builder /go/src/sigs.k8s.io/cloud-provider-azure/bin/azure-cloud-node-manager-${ARCH}.exe /cloud-node-manager.exe ENTRYPOINT ["/cloud-node-manager.exe"] \ No newline at end of file diff --git a/cloud-node-manager-windows.Dockerfile b/cloud-node-manager-windows.Dockerfile index 8bf915fdcf..dcf112b068 100644 --- a/cloud-node-manager-windows.Dockerfile +++ b/cloud-node-manager-windows.Dockerfile @@ -13,10 +13,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - ARG OSVERSION=1809 ARG ARCH=amd64 +# build windows cloud noder manager binary +FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS builder +ARG ENABLE_GIT_COMMAND=true +ARG ARCH +WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure +COPY . . +# Build the Go app +RUN make bin/azure-cloud-node-manager.exe ENABLE_GIT_COMMAND=${ENABLE_GIT_COMMAND} ARCH=${ARCH} + # NOTE(claudiub): Instead of pulling the servercore image, which is ~2GB in side, we # can instead pull the windows-servercore-cache image, which is only a few MBs in size. # The image contains the netapi32.dll we need. @@ -28,6 +36,6 @@ ARG OSVERSION ARG ARCH COPY --from=servercore-helper /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll -COPY bin/azure-cloud-node-manager-${ARCH}.exe /cloud-node-manager.exe +COPY --from=builder /go/src/sigs.k8s.io/cloud-provider-azure/bin/azure-cloud-node-manager-${ARCH}.exe /cloud-node-manager.exe USER ContainerUser ENTRYPOINT ["/cloud-node-manager.exe"] diff --git a/cloud-node-manager.Dockerfile b/cloud-node-manager.Dockerfile index a9e7eac2ee..7af284b59d 100644 --- a/cloud-node-manager.Dockerfile +++ b/cloud-node-manager.Dockerfile @@ -27,8 +27,6 @@ RUN if [ "$ARCH" = "arm64" ] ; then \ WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure COPY . . - -# Build the Go app RUN make bin/azure-cloud-node-manager ENABLE_GIT_COMMAND=${ENABLE_GIT_COMMAND} ARCH=${ARCH} # Use distroless base image for a lean production container. From fda2cd4133441e643ed3f08163af010e8188b3d0 Mon Sep 17 00:00:00 2001 From: Qingchuan Hao Date: Mon, 16 Dec 2024 15:12:50 +0000 Subject: [PATCH 2/3] conditional build stage supports optional local building windows binary --- Makefile | 5 +++++ cloud-node-manager-windows-hpc.Dockerfile | 15 +++++++++++--- cloud-node-manager-windows.Dockerfile | 24 +++++++++++++++++++---- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 3342dd8530..fbc72e9914 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,9 @@ ARCH ?= amd64 WINDOWS_OSVERSION ?= 1809 # The output type for `docker buildx build` could either be docker (local), or registry. OUTPUT_TYPE ?= docker +# LOCAL_WINDOWS_BUILD, when set to true, allows picking a local windows binary built locally. +# This is beneficial when the windows binary requires additional security protection like binary signing. +LOCAL_WINDOWS_BUILD ?=false BASE.windows := mcr.microsoft.com/windows/nanoserver @@ -175,6 +178,7 @@ build-node-image-windows: buildx-setup ## Build node-manager image for Windows. -t $(NODE_MANAGER_WINDOWS_FULL_IMAGE_PREFIX)-$(WINDOWS_OSVERSION)-$(ARCH) \ --build-arg OSVERSION=$(WINDOWS_OSVERSION) \ --build-arg ARCH=$(ARCH) \ + --build-arg LOCAL_BUILD=$(LOCAL_WINDOWS_BUILD) \ -f cloud-node-manager-windows.Dockerfile . \ --provenance=false \ --sbom=false @@ -186,6 +190,7 @@ build-node-image-windows-hpc: buildx-setup ## Build node-manager image for Windo --platform windows/$(ARCH) \ -t $(NODE_MANAGER_WINDOWS_FULL_IMAGE_PREFIX)-hpc-$(ARCH) \ --build-arg ARCH=$(ARCH) \ + --build-arg LOCAL_BUILD=$(LOCAL_WINDOWS_BUILD) \ -f cloud-node-manager-windows-hpc.Dockerfile . \ --provenance=false \ --sbom=false diff --git a/cloud-node-manager-windows-hpc.Dockerfile b/cloud-node-manager-windows-hpc.Dockerfile index 686c0ea85f..be71e60126 100644 --- a/cloud-node-manager-windows-hpc.Dockerfile +++ b/cloud-node-manager-windows-hpc.Dockerfile @@ -13,17 +13,26 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# check cloud-node-manager-windows.Dockerfile for the context of conditional builder image ARG ARCH=amd64 - -# build windows cloud noder manager binary -FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS builder +ARG LOCAL_BUILD=false +# build windows cloud node manager binary +FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS local-build-false ARG ENABLE_GIT_COMMAND=true ARG ARCH WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure COPY . . RUN make bin/azure-cloud-node-manager.exe ENABLE_GIT_COMMAND=${ENABLE_GIT_COMMAND} ARCH=${ARCH} +FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS local-build-true +WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure +COPY . . +COPY bin/azure-cloud-node-manager-*.exe bin/ + +FROM local-build-${LOCAL_BUILD} AS builder + FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0 ARG ARCH +ARG LOCAL_BUILD COPY --from=builder /go/src/sigs.k8s.io/cloud-provider-azure/bin/azure-cloud-node-manager-${ARCH}.exe /cloud-node-manager.exe ENTRYPOINT ["/cloud-node-manager.exe"] \ No newline at end of file diff --git a/cloud-node-manager-windows.Dockerfile b/cloud-node-manager-windows.Dockerfile index dcf112b068..80db6a964c 100644 --- a/cloud-node-manager-windows.Dockerfile +++ b/cloud-node-manager-windows.Dockerfile @@ -15,9 +15,12 @@ # limitations under the License. ARG OSVERSION=1809 ARG ARCH=amd64 +ARG LOCAL_BUILD=false -# build windows cloud noder manager binary -FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS builder +# NOTE(mainred): BuildKit-based builder will skip the used stage depending on the value LOCAL_BUILD. + +# Build windows cloud node manager binary from golang build stage +FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS local-build-false ARG ENABLE_GIT_COMMAND=true ARG ARCH WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure @@ -25,14 +28,27 @@ COPY . . # Build the Go app RUN make bin/azure-cloud-node-manager.exe ENABLE_GIT_COMMAND=${ENABLE_GIT_COMMAND} ARCH=${ARCH} +# COPY the binary built locally to the builder container to normalize the following COPY behavior +FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS local-build-true +WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure +COPY . . +COPY bin/azure-cloud-node-manager-*.exe bin/ + +# mutli-stage dependency are determined before the build starts, so it's invalid to use command like +# `COPY --from=local-build-{LOCAL_BUILD}` to copy the binary from a dynamic source, so we create a +# normalized builder base as a workaround. +# Example of the error: +# ERROR: failed to solve: failed to parse stage name "local-build-{LOCAL_BUILD}": invalid reference format: repository name (library/local-build-{LOCAL_BUILD}) must be lowercase +FROM local-build-${LOCAL_BUILD} AS builder + + # NOTE(claudiub): Instead of pulling the servercore image, which is ~2GB in side, we # can instead pull the windows-servercore-cache image, which is only a few MBs in size. # The image contains the netapi32.dll we need. -FROM --platform=linux/amd64 gcr.io/k8s-staging-e2e-test-images/windows-servercore-cache:1.0-linux-${ARCH}-$OSVERSION as servercore-helper +FROM --platform=linux/amd64 gcr.io/k8s-staging-e2e-test-images/windows-servercore-cache:1.0-linux-${ARCH}-$OSVERSION AS servercore-helper FROM mcr.microsoft.com/windows/nanoserver:$OSVERSION -ARG OSVERSION ARG ARCH COPY --from=servercore-helper /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll From 145d73bf2ac9eb66f7a6d581ae9c4f3b7d3136d6 Mon Sep 17 00:00:00 2001 From: Qingchuan Hao Date: Mon, 23 Dec 2024 12:25:15 +0800 Subject: [PATCH 3/3] Update cloud-node-manager-windows.Dockerfile --- cloud-node-manager-windows.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud-node-manager-windows.Dockerfile b/cloud-node-manager-windows.Dockerfile index 80db6a964c..a902d60698 100644 --- a/cloud-node-manager-windows.Dockerfile +++ b/cloud-node-manager-windows.Dockerfile @@ -17,7 +17,7 @@ ARG OSVERSION=1809 ARG ARCH=amd64 ARG LOCAL_BUILD=false -# NOTE(mainred): BuildKit-based builder will skip the used stage depending on the value LOCAL_BUILD. +# NOTE(mainred): BuildKit-based builder will skip the unused stage determined by the value LOCAL_BUILD. # Build windows cloud node manager binary from golang build stage FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23@sha256:f4fc81062796c14e704559cad3748c5db70bf961ef24d5fac798afa18dff300e AS local-build-false