diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 7622388c1..7859385b1 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -12,6 +12,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +/* groovylint-disable MethodCount */ + import common.IndividualBuildConfig import common.MetaData import common.VersionInfo @@ -1482,6 +1484,15 @@ class Build { context.println ' buildConfig.BUILD_REF: ' + buildConfig.BUILD_REF context.println ' buildConfig.HELPER_REF: ' + buildConfig.HELPER_REF + def openjdk_build_dir = context.WORKSPACE + '/workspace/build/src/build' + def openjdk_build_dir_arg = "" + if (getJavaVersionNumber() >= 21) { + // For reproducible jdk-21+ builds ensure not to build within the openjdk source folder + // so that debug symbols can be reproducibly mapped (https://bugs.openjdk.org/browse/JDK-8326685) + openjdk_build_dir = context.WORKSPACE + '/workspace/build/openjdkbuild' + openjdk_build_dir_arg = " --user-openjdk-build-root-directory ${openjdk_build_dir}" + } + if (cleanWorkspace) { try { try { @@ -1562,9 +1573,9 @@ class Build { context.println "Processing exploded build, sign JMODS, and assemble build, for platform ${buildConfig.TARGET_OS} version ${buildConfig.JAVA_TO_BUILD}" def signBuildArgs if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { - signBuildArgs = env.BUILD_ARGS + ' --make-exploded-image' + signBuildArgs = env.BUILD_ARGS + ' --make-exploded-image' + openjdk_build_dir_arg } else { - signBuildArgs = '--make-exploded-image' + signBuildArgs = '--make-exploded-image' + openjdk_build_dir_arg } context.withEnv(['BUILD_ARGS=' + signBuildArgs]) { context.println 'Building an exploded image for signing' @@ -1675,16 +1686,24 @@ class Build { def assembleBuildArgs if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { - assembleBuildArgs = env.BUILD_ARGS + ' --assemble-exploded-image' + assembleBuildArgs = env.BUILD_ARGS + ' --assemble-exploded-image' + openjdk_build_dir_arg } else { - assembleBuildArgs = '--assemble-exploded-image' + assembleBuildArgs = '--assemble-exploded-image' + openjdk_build_dir_arg } context.withEnv(['BUILD_ARGS=' + assembleBuildArgs]) { context.println 'Assembling the exploded image' context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") } } else { - context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") + def buildArgs + if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { + buildArgs = env.BUILD_ARGS + openjdk_build_dir_arg + } else { + buildArgs = openjdk_build_dir_arg + } + context.withEnv(['BUILD_ARGS=' + buildArgs]) { + context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}") + } } context.println '[CHECKOUT] Reverting pre-build adoptium/temurin-build checkout...' // Special case for the pr tester as checking out to the user's pipelines doesn't play nicely @@ -1700,7 +1719,15 @@ class Build { repoHandler.setUserDefaultsJson(context, DEFAULTS_JSON) repoHandler.checkoutUserBuild(context) printGitRepoInfo() - context.sh(script: "./${DEFAULTS_JSON['scriptDirectories']['buildfarm']}") + def buildArgs + if (env.BUILD_ARGS != null && !env.BUILD_ARGS.isEmpty()) { + buildArgs = env.BUILD_ARGS + openjdk_build_dir_arg + } else { + buildArgs = openjdk_build_dir_arg + } + context.withEnv(['BUILD_ARGS=' + buildArgs]) { + context.sh(script: "./${DEFAULTS_JSON['scriptDirectories']['buildfarm']}") + } context.println '[CHECKOUT] Reverting pre-build user temurin-build checkout...' repoHandler.checkoutUserPipelines(context) printGitRepoInfo() @@ -1766,10 +1793,12 @@ class Build { context.println "Failed to clean ${e}" } } else if (cleanWorkspaceBuildOutputAfter) { - context.println 'Cleaning workspace build output files: ' + context.WORKSPACE + '/workspace/build/src/build' - context.sh(script: 'rm -rf ' + context.WORKSPACE + '/workspace/build/src/build') + context.println 'Cleaning workspace build output files: ' + openjdk_build_dir + context.sh(script: 'rm -rf ' + openjdk_build_dir) context.println 'Cleaning workspace build output files: ' + context.WORKSPACE + '/workspace/target' context.sh(script: 'rm -rf ' + context.WORKSPACE + '/workspace/target') + context.println 'Cleaning workspace build output files: ' + context.WORKSPACE + '/workspace/build/devkit' + context.sh(script: 'rm -rf ' + context.WORKSPACE + '/workspace/build/devkit') } } else { context.println 'Warning: Unable to clean workspace as context.WORKSPACE is null/empty' diff --git a/pipelines/build/devkit/Tools.gmk.patch b/pipelines/build/devkit/Tools.gmk.patch new file mode 100644 index 000000000..84e360386 --- /dev/null +++ b/pipelines/build/devkit/Tools.gmk.patch @@ -0,0 +1,141 @@ +diff --git a/make/devkit/Tools.gmk b/make/devkit/Tools.gmk +index 187320ca2..9f1454d5e 100644 +--- a/make/devkit/Tools.gmk ++++ b/make/devkit/Tools.gmk +@@ -62,6 +62,23 @@ ifeq ($(BASE_OS), OL) + BASE_URL := http://yum.oracle.com/repo/OracleLinux/OL6/4/base/$(ARCH)/ + LINUX_VERSION := OL6.4 + endif ++else ifeq ($(BASE_OS), Centos) ++ DEFAULT_OS_VERSION := 7 ++ ifeq ($(BASE_OS_VERSION), ) ++ BASE_OS_VERSION := $(DEFAULT_OS_VERSION) ++ endif ++ BASE_OS_MAJOR_VERSION := $(shell echo $(BASE_OS_VERSION) | cut -d'.' -f1) ++ ifeq ($(BASE_OS_VERSION),7.9.2009) ++ CENTOS_MIRROR := mirror.centos.org ++ else ++ CENTOS_MIRROR := vault.centos.org ++ endif ++ ifeq ($(ARCH), x86_64) ++ BASE_URL := http://$(CENTOS_MIRROR)/centos/$(BASE_OS_VERSION)/os/$(ARCH)/Packages/ ++ else ++ BASE_URL := http://$(CENTOS_MIRROR)/altarch/$(BASE_OS_VERSION)/os/$(ARCH)/Packages/ ++ endif ++ LINUX_VERSION := Centos$(BASE_OS_VERSION) + else ifeq ($(BASE_OS), Fedora) + ifeq ($(ARCH), riscv64) + DEFAULT_OS_VERSION := rawhide/68692 +@@ -87,6 +104,7 @@ else ifeq ($(BASE_OS), Fedora) + BASE_URL := https://archives.fedoraproject.org/pub/archive/$(FEDORA_TYPE)/releases/$(BASE_OS_VERSION)/Everything/$(ARCH)/os/Packages/ + endif + endif ++ BASE_OS_MAJOR_VERSION := $(shell echo $(BASE_OS_VERSION) | cut -d'.' -f1) + LINUX_VERSION := Fedora_$(BASE_OS_VERSION) + else + $(error Unknown base OS $(BASE_OS)) +@@ -95,16 +113,38 @@ endif + ########################################################################################## + # Define external dependencies + ++# Centos GPG KEYS ++Centos_7_GPG_KEY_aarch64 := EF8F3CA66EFDF32B36CDADF76C7CB6EF305D49D6 #gitleaks:allow ++Centos_7_GPG_KEY_x86_64 := 6341AB2753D78A78A7C27BB124C6A8A7F4A80EB5 #gitleaks:allow ++Centos_7_GPG_KEY_ppc64le := BAFA3436FC50768E3C3C2E4EA963BBDBF533F4FA #gitleaks:allow ++# Fedora GPG KEYS ++Fedora_19_GPG_KEY_s390x := CA81B2C85E4F4D4A1A3F723407477E65FB4B18E6 #gitleaks:allow ++ ++# PGP Signature file extensions ++GCC_SIG := sig ++BINUTILS_SIG := sig ++CCACHE_SIG := asc ++GMP_SIG := sig ++MPC_SIG := sig ++GDB_SIG := sig ++ + # Latest that could be made to work. + GCC_VER := 11.3.0 + ifeq ($(GCC_VER), 11.3.0) + gcc_ver := gcc-11.3.0 ++ GCC_GPG_KEY := 7F74F97C103468EE5D750B583AB00996FC26A641 #gitleaks:allow + binutils_ver := binutils-2.39 ++ BINUTILS_GPG_KEY := 3A24BC1E8FB409FA9F14371813FCEF89DD9E3C4F #gitleaks:allow + ccache_ver := ccache-3.7.12 ++ CCACHE_GPG_KEY := 5A939A71A46792CF57866A51996DDA075594ADB8 #gitleaks:allow + mpfr_ver := mpfr-4.1.1 ++ MPFR_SHA256 := 85fdf11614cc08e3545386d6b9c8c9035e3db1e506211a45f4e108117fe3c951 #gitleaks:allow + gmp_ver := gmp-6.2.1 ++ GMP_GPG_KEY := 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 #gitleaks:allow + mpc_ver := mpc-1.2.1 ++ MPC_GPG_KEY := AD17A21EF8AED8F1CC02DBD9F7D5C9BF765C61E3 #gitleaks:allow + gdb_ver := gdb-11.2 ++ GDB_GPG_KEY := F40ADB902B24264AA42E50BF92EDB04BFF325CF3 #gitleaks:allow + REQUIRED_MIN_MAKE_MAJOR_VERSION := 4 + else ifeq ($(GCC_VER), 11.2.0) + gcc_ver := gcc-11.2.0 +@@ -246,7 +286,13 @@ download-rpms: + # Only run this if rpm dir is empty. + ifeq ($(wildcard $(DOWNLOAD_RPMS)/*.rpm), ) + cd $(DOWNLOAD_RPMS) && \ +- wget -r -np -nd $(patsubst %, -A "*%*.rpm", $(RPM_LIST)) $(BASE_URL) ++ wget -e robots=off -r -np -nd $(patsubst %, -A "*%*.rpm", $(RPM_LIST)) $(BASE_URL) ++ gpg --keyserver keyserver.ubuntu.com --recv-keys \ ++ $($(BASE_OS)_$(BASE_OS_MAJOR_VERSION)_GPG_KEY_$(ARCH)) ++ rm -f $(DOWNLOAD_RPMS)/rpm.sig && gpg --armor --export \ ++ $($(BASE_OS)_$(BASE_OS_MAJOR_VERSION)_GPG_KEY_$(ARCH)) > $(DOWNLOAD_RPMS)/rpm.sig ++ rpm --import $(DOWNLOAD_RPMS)/rpm.sig || true ++ rpm -K $(DOWNLOAD_RPMS)/*.rpm + endif + + ########################################################################################## +@@ -271,6 +317,14 @@ define Download + + $$($(1)_FILE) : + wget -P $(DOWNLOAD) $$($(1)) ++ ifeq ($(1),MPFR) ++ echo $($(1)_SHA256) $$@ | sha256sum -c ++ else ++ wget -P $(DOWNLOAD) $$($(1)).$($(1)_SIG) ++ gpg --keyserver keyserver.ubuntu.com --recv-keys $($(1)_GPG_KEY) ++ echo -e "5\ny\n" | gpg --batch --command-fd 0 --expert --edit-key $($(1)_GPG_KEY) trust; ++ gpg --verify $$@.$($(1)_SIG) $$@ ++ endif + endef + + # Download and unpack all source packages +@@ -323,6 +377,9 @@ $(foreach p,$(RPM_FILE_LIST),$(eval $(call unrpm,$(p)))) + # have it anyway, but just to make sure... + # Patch libc.so and libpthread.so to force linking against libraries in sysroot + # and not the ones installed on the build machine. ++# Remove comment sections from static libraries and C runtime objects ++# to prevent leaking RHEL-specific package versions into ++# devkit-produced binaries. + $(libs) : $(rpms) + @echo Patching libc and pthreads + @(for f in `find $(SYSROOT) -name libc.so -o -name libpthread.so`; do \ +@@ -332,6 +389,7 @@ $(libs) : $(rpms) + -e 's|/lib/||g' ) > $$f.tmp ; \ + mv $$f.tmp $$f ; \ + done) ++ @find $(SYSROOT) -name '*.[ao]' -exec objcopy --remove-section .comment '{}' ';' + @mkdir -p $(SYSROOT)/usr/lib + @touch $@ + +@@ -440,6 +498,9 @@ endif + + # Makefile creation. Simply run configure in build dir. + # Setting CFLAGS to -O2 generates a much faster ld. ++# Use --enable-deterministic-archives so that make targets that ++# generate "ar" archives, such as "static-libs-image", produce ++# deterministic .a files. + $(bfdmakes) \ + $(BUILDDIR)/$(binutils_ver)/Makefile \ + : $(BINUTILS_CFG) +@@ -454,6 +515,7 @@ $(BUILDDIR)/$(binutils_ver)/Makefile \ + --with-sysroot=$(SYSROOT) \ + --disable-nls \ + --program-prefix=$(TARGET)- \ ++ --enable-deterministic-archives \ + --enable-multilib \ + --enable-threads \ + --enable-plugins \ diff --git a/pipelines/build/devkit/binutils-2.39.patch b/pipelines/build/devkit/binutils-2.39.patch new file mode 100644 index 000000000..0824027a7 --- /dev/null +++ b/pipelines/build/devkit/binutils-2.39.patch @@ -0,0 +1,38 @@ +diff --git a/gprofng/Makefile.am b/gprofng/Makefile.am +index 2fcd695140ca6e3f06f3cc6844d498847a21a63d..96f8015700885e00b9e121cd61adede63c2b832e 100644 (file) +--- a/gprofng/Makefile.am ++++ b/gprofng/Makefile.am +@@ -23,11 +23,14 @@ AUTOMAKE_OPTIONS = dejagnu foreign + if BUILD_COLLECTOR + COLLECTOR_SUBDIRS = libcollector + endif ++if BUILD_MAN ++ DOC_SUBDIR = doc ++endif + if BUILD_SRC +- SRC_SUBDIRS = src gp-display-html doc ++ SRC_SUBDIRS = src gp-display-html $(DOC_SUBDIR) + endif + SUBDIRS = $(COLLECTOR_SUBDIRS) $(SRC_SUBDIRS) +-DIST_SUBDIRS = libcollector src gp-display-html doc ++DIST_SUBDIRS = libcollector src gp-display-html $(DOC_SUBDIR) + + # Setup the testing framework, if you have one + EXPECT = expect +diff --git a/gprofng/Makefile.in b/gprofng/Makefile.in +index fd5279b4df13ca21061bfc833797a0bb6ada62bc..d660f99233fedef9df9cae31629cd3ba57a51150 100644 (file) +--- a/gprofng/Makefile.in ++++ b/gprofng/Makefile.in +@@ -381,9 +381,10 @@ zlibinc = @zlibinc@ + ACLOCAL_AMFLAGS = -I . -I .. + AUTOMAKE_OPTIONS = dejagnu foreign + @BUILD_COLLECTOR_TRUE@COLLECTOR_SUBDIRS = libcollector +-@BUILD_SRC_TRUE@SRC_SUBDIRS = src gp-display-html doc ++@BUILD_MAN_TRUE@DOC_SUBDIR = doc ++@BUILD_SRC_TRUE@SRC_SUBDIRS = src gp-display-html $(DOC_SUBDIR) + SUBDIRS = $(COLLECTOR_SUBDIRS) $(SRC_SUBDIRS) +-DIST_SUBDIRS = libcollector src gp-display-html doc ++DIST_SUBDIRS = libcollector src gp-display-html $(DOC_SUBDIR) + RUNTEST = runtest + RUNTESTFLAGS = + BASEDIR = $(srcdir)/.. diff --git a/pipelines/build/devkit/build_devkit.groovy b/pipelines/build/devkit/build_devkit.groovy new file mode 100644 index 000000000..6c8f125f8 --- /dev/null +++ b/pipelines/build/devkit/build_devkit.groovy @@ -0,0 +1,133 @@ +package common +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + * Builds a linux-gnu DevKit for the given openjdk version and architecture. + * Supported architectures: + * x86_64 + * aarch64 + * ppc64le + * s390x + */ + +// The devkit release tag this build will get published under +// groovylint-disable-next-line +def adoptium_devkit_release_tag + +def build_devkit() { + stage('Build DevKit') { + // Make DevKit + sh("cd pipelines/build/devkit && ./make_devkit.sh ${params.VERSION} ${params.ARCH} ${params.BASE_OS} ${params.BASE_OS_VERSION}") + + def devkit_target="${params.ARCH}-linux-gnu" + + // Get gcc version and base OS from devkit.info + def gcc_ver=sh(script:'grep DEVKIT_NAME pipelines/build/devkit/'+params.VERSION+'/build/devkit/result/'+devkit_target+'-to-'+devkit_target+'/devkit.info | cut -d"=" -f2 | tr -d "\\" \\n"', returnStdout: true) + + // The devkit release tag this build will get published under + adoptium_devkit_release_tag = "${gcc_ver}-${params.BUILD}" + + def adoptium_devkit_filename = "devkit-${adoptium_devkit_release_tag}-${devkit_target}" + + // Store Adoptium metadata within the devkit.info file + sh(script:"echo ADOPTIUM_DEVKIT_RELEASE=\"${adoptium_devkit_release_tag}\" >> pipelines/build/devkit/${params.VERSION}/build/devkit/result/${devkit_target}-to-${devkit_target}/devkit.info") + sh(script:"echo ADOPTIUM_DEVKIT_TARGET=\"${devkit_target}\" >> pipelines/build/devkit/${params.VERSION}/build/devkit/result/${devkit_target}-to-${devkit_target}/devkit.info") + + def devkit_tarball = "${adoptium_devkit_filename}.tar.xz" + println "devkit artifact filename = ${devkit_tarball}" + + // Compress and archive + sh(script:"tar -cf - -C pipelines/build/devkit/${params.VERSION}/build/devkit/result/${devkit_target}-to-${devkit_target} . | GZIP=-9 xz -c > ${devkit_tarball}") + + // Create sha256.txt + sh(script:"sha256sum ${devkit_tarball} > ${devkit_tarball}.sha256.txt") + + sh(script:"mkdir workspace && mv ${devkit_tarball}* workspace") + archiveArtifacts artifacts: "workspace/*" + } +} + +def gpgSign() { + stage('GPG sign') { + def params = [ + string(name: 'UPSTREAM_JOB_NUMBER', value: "${env.BUILD_NUMBER}"), + string(name: 'UPSTREAM_JOB_NAME', value: "${env.JOB_NAME}"), + string(name: 'UPSTREAM_DIR', value: 'workspace') + ] + + def signSHAsJob = build job: 'build-scripts/release/sign_temurin_gpg', + propagate: true, + parameters: params + + copyArtifacts( + projectName: 'build-scripts/release/sign_temurin_gpg', + selector: specific("${signSHAsJob.getNumber()}"), + filter: '**/*.sig', + fingerprintArtifacts: true, + target: 'workspace', + flatten: true) + + archiveArtifacts artifacts: "workspace/*.sig" + } +} + +def dryrunPublish() { + stage('Dry run publish') { + println "Running a DRY_RUN publish_devkit_tool for tag: " + adoptium_devkit_release_tag + + def params = [ + string(name: 'TAG', value: "${adoptium_devkit_release_tag}"), + string(name: 'UPSTREAM_JOB_NUMBER', value: "${env.BUILD_NUMBER}"), + string(name: 'UPSTREAM_JOB_NAME', value: "${env.JOB_NAME}"), + string(name: 'ARTIFACTS_TO_COPY', value: 'workspace/*.tar.xz,workspace/*.sha256.txt,workspace/*.sig'), + booleanParam(name: 'DRY_RUN', value: true) + ] + + build job: 'build-scripts/release/publish_devkit_tool', + propagate: true, + parameters: params + } +} + +def build() { + // Checkout pipelines code + checkout scm + + build_devkit() + + gpgSign() + + dryrunPublish() +} + +node(params.DEVKIT_BUILD_NODE) { + try { + cleanWs notFailBuild: true, disableDeferredWipeout: true, deleteDirs: true + + if (params.DOCKER_IMAGE != "") { + // Build within docker container + docker.image(params.DOCKER_IMAGE).pull() + docker.image(params.DOCKER_IMAGE).inside() { + build() + } + } else { + // Build directly on host + build() + } + } finally { + cleanWs notFailBuild: true + } +} + diff --git a/pipelines/build/devkit/make_devkit.sh b/pipelines/build/devkit/make_devkit.sh new file mode 100755 index 000000000..7560c2d18 --- /dev/null +++ b/pipelines/build/devkit/make_devkit.sh @@ -0,0 +1,53 @@ +#!/bin/bash +################################################################################ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +set -e + +if [ $# -lt 4 ]; then + echo "Usage: $0 VERSION ARCH BASE_OS BASE_OS_VERSION" + echo "eg: $0 jdk21u aarch64 Centos 7.6.1810" + exit 1 +fi + +VERSION=$1 +ARCH=$2 +BASE_OS=$3 +BASE_OS_VERSION=$4 + +# Create temp GPG home +GNUPGHOME="$(mktemp -d /tmp/.gpg-temp.XXXXXX)" +chmod 700 ${GNUPGHOME} +export GNUPGHOME + +openjdkRepo="https://github.com/openjdk/${VERSION}.git" + +# Clone upstream openjdk repo +git clone --depth 1 ${openjdkRepo} ${VERSION} +cd ${VERSION} + +# Patch to support Centos7 +cp ../binutils-2.39.patch make/devkit/patches/${ARCH}-binutils-2.39.patch +patch -p1 < ../Tools.gmk.patch + +devkit_target="${ARCH}-linux-gnu" + +# Perform devkit build +cd make/devkit && make TARGETS=${devkit_target} BASE_OS=${BASE_OS} BASE_OS_VERSION=${BASE_OS_VERSION} + +# Back to original folder +cd ../../.. + +echo "DevKit build successful: ${VERSION}/build/devkit/result/${devkit_target}-to-${devkit_target}" +