From 7b097cf54e9e8b6101186c0704ec83fa110d729a Mon Sep 17 00:00:00 2001 From: Sumit Garg Date: Tue, 17 Dec 2024 11:21:16 -0500 Subject: [PATCH 1/6] Delete hanging space at end of line --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7baab148c1fd8..70c941f275d31 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ GOOS ?= $(shell go env GOOS) GOARCH ?= $(shell go env GOARCH) HOSTGO := env -u GOOS -u GOARCH -u GOARM -- go -LDFLAGS := $(LDFLAGS) -X main.commit=$(commit) -X main.branch=$(branch) -X main.goos=$(GOOS) -X main.goarch=$(GOARCH) +LDFLAGS := $(LDFLAGS) -X main.commit=$(commit) -X main.branch=$(branch) -X main.goos=$(GOOS) -X main.goarch=$(GOARCH) ifeq ($(GOARCH), arm) LDFLAGS := $(LDFLAGS) -X main.goarm=$(GOARM) endif From 0ca58832d3085f7f38a04e01e91a5e8eb132bcc9 Mon Sep 17 00:00:00 2001 From: Sumit Garg Date: Tue, 17 Dec 2024 14:43:20 -0500 Subject: [PATCH 2/6] Populate go cache under $PWD/.tmp --- build.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/build.sh b/build.sh index a4798b811cd5d..bf124ceb6a34c 100755 --- a/build.sh +++ b/build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Simple ExtremeNetworks script to facilitate building and # uploading the telegraf utility. @@ -10,8 +10,6 @@ # set -e -export LDFLAGS="-w -s" - # Default ARM type for arm architecture arm_type=5 @@ -25,9 +23,20 @@ usage() build() { + local _go_opts="" + + _go_opts+="${_go_opts:+ }GOOS=linux" + _go_opts+="${_go_opts:+ }GOARCH=${bld_arch}" + _go_opts+="${_go_opts:+ }GOARM=${arm_type}" + _go_opts+="${_go_opts:+ }GOPROXY=https://proxy.golang.org,direct" + _go_opts+="${_go_opts:+ }GOSUMDB=sum.golang.org" + _go_opts+="${_go_opts:+ }CGO_ENABLED=0" + _go_opts+="${_go_opts:+ }GOCACHE=${gocache@Q}" + _go_opts+="${_go_opts:+ }GOMODCACHE=${gomodcache@Q}" + make clean rm -f ${target} - make CGO_ENABLED=0 GOOS=linux GOARCH=${bld_arch} GOARM=${arm_type} GOPROXY=https://proxy.golang.org,direct GOSUMDB=sum.golang.org + LDFLAGS="-w -s" make ${_go_opts} telegraf tar -cf ${target} telegraf MIT generic_MIT rm -f telegraf } @@ -65,8 +74,8 @@ if [[ -z "$1" || "$1" == "--help" || "$1" == "-h" || "$1" == "?" ]]; then fi # force go to use alternate location of modules -go env -w GOMODCACHE=/opt/go/pkg/mod -go env -w GOCACHE=$(pwd -P)/.cache/go-build +gocache="$(pwd -P)/.tmp/go-cache" +gomodcache="$(pwd -P)/.tmp/go-mod" # grab version strings telegraf_version=$(cat build_version.txt) extr_version=$(cat extr_version.txt) From 0d1cf7bd844a79c153ce18f3de3011442cd4b61e Mon Sep 17 00:00:00 2001 From: Sumit Garg Date: Thu, 19 Dec 2024 12:04:24 -0500 Subject: [PATCH 3/6] Refactor build.sh script; no functional change --- build.sh | 182 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 106 insertions(+), 76 deletions(-) diff --git a/build.sh b/build.sh index bf124ceb6a34c..bf42aaf152f3e 100755 --- a/build.sh +++ b/build.sh @@ -8,107 +8,137 @@ # The extr_version is the version used when we make Extreme specific changes on # a particular telegraf branch. # -set -e -# Default ARM type for arm architecture -arm_type=5 - -usage() +show_usage() { - echo "usage: $0 arch {build | upload}" - echo " . arch : valid architectures: arm, arm64, x86_64, mips" - echo " . build : build and tar utility for specified architecture" - echo " . upload: upload specified architecture's tar to Artifactory" + echo "usage: ${0} {build | upload}" + echo " . arch : valid architectures: arm, arm64, x86_64, mips" + echo " . build : build and tar utility for specified architecture" + echo " . upload : upload specified architecture's tar to Artifactory" } -build() +do_build() { - local _go_opts="" + local _extr_arch="${1}" + local _target_tarball="${2}" + + # set the target architecture for the executable + local _target_arch="${_extr_arch}" + if [[ "${_target_arch}" = "x86_64" ]]; then + _target_arch="amd64" + fi + # options passed to GO + local _go_opts="" _go_opts+="${_go_opts:+ }GOOS=linux" - _go_opts+="${_go_opts:+ }GOARCH=${bld_arch}" - _go_opts+="${_go_opts:+ }GOARM=${arm_type}" - _go_opts+="${_go_opts:+ }GOPROXY=https://proxy.golang.org,direct" - _go_opts+="${_go_opts:+ }GOSUMDB=sum.golang.org" - _go_opts+="${_go_opts:+ }CGO_ENABLED=0" - _go_opts+="${_go_opts:+ }GOCACHE=${gocache@Q}" - _go_opts+="${_go_opts:+ }GOMODCACHE=${gomodcache@Q}" + _go_opts+="${_go_opts:+ }GOARCH=${_target_arch}" + if [[ "${_target_arch}" == "arm" ]]; then + # set GOARM only when building for 32 bit ARM + _go_opts+="${_go_opts:+ }GOARM=${arm32_type}" + fi + + # force GO to use alternate location of cache and modules + local _gocache="$(pwd -P)/.tmp/go-cache" + local _gomodcache="$(pwd -P)/.tmp/go-mod" + _go_opts+="${_go_opts:+ }GOCACHE=${_gocache@Q}" + _go_opts+="${_go_opts:+ }GOMODCACHE=${_gomodcache@Q}" make clean - rm -f ${target} - LDFLAGS="-w -s" make ${_go_opts} telegraf - tar -cf ${target} telegraf MIT generic_MIT + rm -f "${_target_tarball}" + env LDFLAGS="-w -s" CGO_ENABLED=0 ${_go_opts} make telegraf + tar -cf "${_target_tarball}" telegraf MIT generic_MIT rm -f telegraf } -upload() +do_upload() { - if [ ! -f ${target} ]; then - echo "info: ${target} not found; building first..." - build - if [ ! -f ${target} ]; then - echo "error: could not find or build '${target}' tarball" + local _extr_arch="${1}" + local _target_tarball="${2}" + + if [[ ! -f "${_target_tarball}" ]]; then + echo "info: ${_target_tarball} not found; building first..." + do_build "${_extr_arch}" "${_target_tarball}" + if [[ ! -f "${_target_tarball}" ]]; then + echo "error: could not find or build '${_target_tarball}' tarball" exit 1 fi fi # make sure jfrog config is set up - if ! jfrog config show ${salem} &>/dev/null ; then - echo "Could not find Salem Artifactory config. Let's create one..." - echo "Accept defaults and use your corporate password."; echo "" - jfrog config add --url http://engartifacts1.extremenetworks.com:8081 --user $(whoami) ${salem} - if ! jfrog config show ${salem} &>/dev/null ; then - echo "error: failed to configure ${salem} Artifactory server access" + if ! jfrog config show "${afy_server_name}" &>/dev/null ; then + echo "Could not find ${afy_server_name} Artifactory config. Let's create one..." + echo "Accept defaults and use your corporate password."; echo + jfrog config add --url "${afy_server_url}" --user "${USER}" "${afy_server_name}" + if ! jfrog config show "${afy_server_name}" &>/dev/null ; then + echo "error: failed to configure ${afy_server_name} Artifactory server access" exit 1 fi fi - jfrog rt upload ${target} xos-binaries-local-release/telegraf/${arch}/${target} --server-id ${salem} + + jfrog rt upload "${_target_tarball}" "${aft_repo}/${_extr_arch}/${_target_tarball}" --server-id "${afy_server_name}" } ####################### # execution starts here ####################### -if [[ -z "$1" || "$1" == "--help" || "$1" == "-h" || "$1" == "?" ]]; then - usage - exit 0 -fi - -# force go to use alternate location of modules -gocache="$(pwd -P)/.tmp/go-cache" -gomodcache="$(pwd -P)/.tmp/go-mod" -# grab version strings -telegraf_version=$(cat build_version.txt) -extr_version=$(cat extr_version.txt) -salem=Salem - -# set architecture name and build architecture as used by golang -arch=$1 -if [ "$arch" = "x86_64" ]; then - bld_arch=amd64 -else - bld_arch=${arch} -fi -target=telegraf_${arch}_${telegraf_version}.${extr_version}.tar - -# check action argument -case $2 in - build | upload) - action=$2 - ;; - *) - echo "error: invalid action '$2'" - usage - exit 1 -esac - -# perform action -case $1 in - arm64 | mips | x86_64 | arm) - $action - ;; - *) - echo "error: invalid architecture '$1'" - usage + +main() +{ + # all hard-coded globals go here + arm32_type="5" # default ARM type for "arm" architecture + afy_server_name="Salem" + afy_server_url="http://engartifacts1.extremenetworks.com:8081" + aft_repo="xos-binaries-local-release/telegraf" + + local _extr_arch="${1}" + local _script_action="${2}" + + # verify inputs - count of args + if [[ ${#} -ne 2 ]]; then + echo "error: incorrect number of arguments '${#}'" + show_usage exit 1 -esac + fi + + # verify inputs - architecture + case "${_extr_arch}" in + arm64 | mips | x86_64 | arm) + : + ;; + --help | -h | -?) + show_usage + exit 0 + ;; + *) + echo "error: invalid architecture '${1}'" + show_usage + exit 1 + esac + + # verify inputs - action + case "${_script_action}" in + build | upload) + : + ;; + --help | -h | -?) + show_usage + exit 0 + ;; + *) + echo "error: invalid action '$2'" + show_usage + exit 1 + esac + + # grab version strings and set the name of the target tarball + local _telegraf_version="$(< build_version.txt)" + local _extr_version="$(< extr_version.txt)" + local _target_tarball="telegraf_${_extr_arch}_${_telegraf_version}.${_extr_version}.tar" + + "do_${_script_action}" "${_extr_arch}" "${_target_tarball}" +} + +set -o errexit +set -o pipefail +main "${@}" From cbee2bbbffd1120792d4a6d4b272eb5dfad3db2a Mon Sep 17 00:00:00 2001 From: Sumit Garg Date: Thu, 26 Dec 2024 19:26:25 -0500 Subject: [PATCH 4/6] Build using a docker container --- Dockerfile | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ build.sh | 61 +++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 121 insertions(+), 10 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000..7b8a59cc3c4f7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,70 @@ +# +# Dockerfile to build telegraf in a container. +# + +# +# Global build args that can be passed from the builder: +# +ARG BASE_GOLANG_VER="1.21" + +ARG TELEGRAF_BUILD_ROOT="/root/go/telegraf" + +# Typically set to any GO specific environment variables; see build.sh for how +# this is used. +ARG BUILD_GO_OPTS= + + +# Start with the base golang container and get a custom golang image. +FROM "golang:${BASE_GOLANG_VER}-bookworm" AS golang + RUN ln -sf /bin/bash /bin/sh + + +# Copy telegraf sources into the golang build environment. +FROM golang AS source + ARG TELEGRAF_BUILD_ROOT + + WORKDIR "${TELEGRAF_BUILD_ROOT}" + + COPY .git .git + COPY agent agent + COPY cmd cmd + COPY config config + COPY filter filter + COPY internal internal + COPY logger logger + COPY metric metric + COPY models models + COPY plugins plugins + COPY selfstat selfstat + COPY *.go go.* ./ + COPY build_version.txt ./ + COPY Makefile ./ + + +# Build telegraf using the custom golang container. +FROM source AS build + ARG BUILD_GO_OPTS + + RUN \ + export LDFLAGS="-w -s" CGO_ENABLED=0 ${BUILD_GO_OPTS} \ + && \ + git config --global user.email "dev@extremenetworks.com" \ + && \ + git config --global user.name "Dev Extreme" \ + && \ + export \ + && \ + make telegraf && \ + : + + +# Copy the binary to an empty container. +FROM scratch AS binary + ARG TELEGRAF_BUILD_ROOT + + WORKDIR / + + COPY --from=build "${TELEGRAF_BUILD_ROOT}/telegraf" /usr/bin/ + + ENTRYPOINT ["/usr/bin/telegraf"] + CMD ["--help"] diff --git a/build.sh b/build.sh index bf42aaf152f3e..dfa80257f164b 100755 --- a/build.sh +++ b/build.sh @@ -30,22 +30,63 @@ do_build() # options passed to GO local _go_opts="" - _go_opts+="${_go_opts:+ }GOOS=linux" - _go_opts+="${_go_opts:+ }GOARCH=${_target_arch}" if [[ "${_target_arch}" == "arm" ]]; then # set GOARM only when building for 32 bit ARM _go_opts+="${_go_opts:+ }GOARM=${arm32_type}" + elif [[ "${_target_arch}" == "mips" ]]; then + # set GOARCH to build a 32 bit executable for MIPS + _go_opts+="${_go_opts:+ }GOARCH=${_target_arch}" fi - # force GO to use alternate location of cache and modules - local _gocache="$(pwd -P)/.tmp/go-cache" - local _gomodcache="$(pwd -P)/.tmp/go-mod" - _go_opts+="${_go_opts:+ }GOCACHE=${_gocache@Q}" - _go_opts+="${_go_opts:+ }GOMODCACHE=${_gomodcache@Q}" - - make clean rm -f "${_target_tarball}" - env LDFLAGS="-w -s" CGO_ENABLED=0 ${_go_opts} make telegraf + + # + # REF: https://docs.docker.com/build/building/multi-platform/#qemu + # + # We do not leverage the cross compilation feature of golang to build the executable; + # in other words, GOOS and GOARCH are (typically) not passed to the go compiler. + # Instead we build on a node (i.e. container) that has the same CPU as that target; + # in other words: + # arch of container running golang compiler + # == target arch + # This node can be one of the following: + # 1. the host on which docker is running (typically amd64 or arm64) i.e. + # arch of host running the golang containers used to build the executable + # == arch of container running golang compiler + # 2. a container running under qemu (qemu is provided by the host) i.e. + # arch of host running the golang containers used to build the executable + # != arch of container running golang compiler + # + # For #2 above, the container effectively runs under qemu. You might need to run + # the following command to install qemu on the host: + # docker run --privileged --rm tonistiigi/binfmt --install linux/${_container_arch} + # + # NOTE: We are not building a multi-platform image as that requires some capabilities + # in the image store that might/might not be present in the docker install. + # + + # architecture of the container to use for building + local _container_arch="${_target_arch}" + if [[ "${_container_arch}" == "mips" ]]; then + # use "mips64le" for the container arch when target arch is "mips" + _container_arch="mips64le" + fi + + local _dockerfile_stage="binary" + local _target_image="telegraf/${_dockerfile_stage}/${_target_arch}:$(git describe --dirty)" + docker buildx build --progress plain \ + --build-arg BUILD_GO_OPTS="${_go_opts}" \ + --platform "linux/${_container_arch}" \ + --tag "${_target_image}" \ + --target "${_dockerfile_stage}" \ + . + + local _copy_container="$(docker container create --quiet "${_target_image}")" + docker container cp "${_copy_container}:/usr/bin/telegraf" telegraf + docker container rm "${_copy_container}" + + docker image rm "${_target_image}" + tar -cf "${_target_tarball}" telegraf MIT generic_MIT rm -f telegraf } From 492e88ea915b0af521809b88f4243702433fee43 Mon Sep 17 00:00:00 2001 From: Sumit Garg Date: Fri, 27 Dec 2024 15:26:18 -0500 Subject: [PATCH 5/6] Build FIPS capable executables --- Dockerfile | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- build.sh | 8 ++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7b8a59cc3c4f7..368985993a3bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,9 +6,16 @@ # Global build args that can be passed from the builder: # ARG BASE_GOLANG_VER="1.21" +ARG FIPS_GOLANG_VER="${BASE_GOLANG_VER}" +ARG GOLANG_FIPS_BUILD_ROOT="/root/go/golang-fips" ARG TELEGRAF_BUILD_ROOT="/root/go/telegraf" +# When BUILD_GO_FIPS=1, build a dynamically linked executable using go patched +# with golang-fips patch and cgo (CGO_ENABLED=1). Otherwise, build a statically +# linked executable with the standard go. +ARG BUILD_GO_FIPS= + # Typically set to any GO specific environment variables; see build.sh for how # this is used. ARG BUILD_GO_OPTS= @@ -16,7 +23,27 @@ ARG BUILD_GO_OPTS= # Start with the base golang container and get a custom golang image. FROM "golang:${BASE_GOLANG_VER}-bookworm" AS golang + ARG \ + BASE_GOLANG_VER \ + FIPS_GOLANG_VER \ + GOLANG_FIPS_BUILD_ROOT \ + BUILD_GO_FIPS + ARG GO_SRC_BRANCH="release-branch.go${FIPS_GOLANG_VER}" + ARG GOLANG_FIPS_BRANCH="go${FIPS_GOLANG_VER}-fips-release" + + ADD --keep-git-dir=true "https://github.com/golang-fips/go.git#${GOLANG_FIPS_BRANCH}" "${GOLANG_FIPS_BUILD_ROOT}" + RUN ln -sf /bin/bash /bin/sh + RUN \ + if [[ "${BUILD_GO_FIPS}" == "1" ]]; then \ + git config --global user.email "dev@extremenetworks.com" && \ + git config --global user.name "Dev Extreme" && \ + cd "${GOLANG_FIPS_BUILD_ROOT}" && \ + ./scripts/full-initialize-repo.sh "${GO_SRC_BRANCH}" && \ + : ; \ + fi \ + && \ + : # Copy telegraf sources into the golang build environment. @@ -43,16 +70,35 @@ FROM golang AS source # Build telegraf using the custom golang container. FROM source AS build - ARG BUILD_GO_OPTS + ARG \ + GOLANG_FIPS_BUILD_ROOT \ + BUILD_GO_FIPS \ + BUILD_GO_OPTS RUN \ - export LDFLAGS="-w -s" CGO_ENABLED=0 ${BUILD_GO_OPTS} \ + export LDFLAGS="-w -s" ${BUILD_GO_OPTS} \ && \ git config --global user.email "dev@extremenetworks.com" \ && \ git config --global user.name "Dev Extreme" \ && \ export \ + && \ + if [[ "${BUILD_GO_FIPS}" == "1" ]]; then \ + apt-get update --yes && \ + apt-get install --yes --no-install-recommends --no-install-suggests libssl-dev && \ + if [[ "${GOARCH}" == "mips" ]]; then \ + apt-get install --yes --no-install-recommends --no-install-suggests gcc-multilib && \ + : ; \ + fi && \ + rm -rf /var/lib/apt/lists/* && \ + export PATH="${GOLANG_FIPS_BUILD_ROOT}/bin:${PATH}" && \ + export CGO_ENABLED=1 && \ + : ; \ + else \ + export CGO_ENABLED=0 && \ + : ; \ + fi \ && \ make telegraf && \ : diff --git a/build.sh b/build.sh index dfa80257f164b..13f750edcedf9 100755 --- a/build.sh +++ b/build.sh @@ -28,6 +28,13 @@ do_build() _target_arch="amd64" fi + # default to building a FIPS executable + local _go_fips=1 + if [[ "${_target_arch}" == "arm" || "${_target_arch}" == "mips" ]]; then + # except for 32 bit ARM and MIPS + _go_fips=0 + fi + # options passed to GO local _go_opts="" if [[ "${_target_arch}" == "arm" ]]; then @@ -75,6 +82,7 @@ do_build() local _dockerfile_stage="binary" local _target_image="telegraf/${_dockerfile_stage}/${_target_arch}:$(git describe --dirty)" docker buildx build --progress plain \ + --build-arg BUILD_GO_FIPS="${_go_fips}" \ --build-arg BUILD_GO_OPTS="${_go_opts}" \ --platform "linux/${_container_arch}" \ --tag "${_target_image}" \ From 60edcd9abf8e70b25c32ee82677210db78a353d3 Mon Sep 17 00:00:00 2001 From: Sumit Garg Date: Mon, 30 Dec 2024 12:13:01 -0500 Subject: [PATCH 6/6] Bump extreme version --- extr_version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extr_version.txt b/extr_version.txt index 48082f72f087c..b1bd38b62a080 100644 --- a/extr_version.txt +++ b/extr_version.txt @@ -1 +1 @@ -12 +13