Enabling a new DevKit build job and support use within pipeline configuration

pipelines/build/common/build_base_file.groovy

@@ -57,6 +57,7 @@ class Builder implements Serializable {
     boolean aqaAutoGen
     String publishName
     String additionalConfigureArgs
+    String devkit
     def scmVars
     String additionalBuildArgs
     String overrideFileNameVersion
@@ -127,6 +128,9 @@ class Builder implements Serializable {
         if (additionalBuildArgs) {
             buildArgs += ' ' + additionalBuildArgs
         }
+
+        def devkit = getDevkit(platformConfig, variant)
+
         def enableReproducibleCompare = isEnableReproducibleCompare(platformConfig, variant)
         def testList = getTestList(platformConfig, variant)
 
@@ -169,6 +173,7 @@ class Builder implements Serializable {
             AQA_REF: aqaReference,
             AQA_AUTO_GEN: aqaAutoGen,
             BUILD_ARGS: buildArgs,
+            DEVKIT: devkit,
             NODE_LABEL: "${additionalNodeLabels}&&${platformConfig.os}&&${archLabel}",
             ADDITIONAL_TEST_LABEL: "${additionalTestLabels}",
             KEEP_TEST_REPORTDIR: keepTestReportDir,
@@ -558,6 +563,21 @@ class Builder implements Serializable {
         return platformSpecificConfigPath
     }
 
+    /*
+    Get the devkit platform configuration value
+    */
+    def getDevkit(Map<String, ?> configuration, String variant) {
+        def devkit = ''
+        if (configuration.containsKey('devkit')) {
+            if (isMap(configuration.devkit)) {
+                devkit = (configuration.devkit as Map<String, ?>).get(variant)
+            } else {
+                devkit = configuration.devkit
+            }
+        }   
+        return devkit
+    }
+
     /*
     Constructs any necessary additional build labels from the build configurations.
     This builds up a node param string that defines what nodes are eligible to run the given job.
@@ -1040,6 +1060,7 @@ return {
     String overridePublishName,
     String useAdoptShellScripts,
     String additionalConfigureArgs,
+    String devkit,
     def scmVars,
     String additionalBuildArgs,
     String overrideFileNameVersion,
@@ -1106,6 +1127,7 @@ return {
             aqaAutoGen: Boolean.parseBoolean(aqaAutoGen),
             publishName: publishName,
             additionalConfigureArgs: additionalConfigureArgs,
+            devkit: devkit,
             scmVars: scmVars,
             additionalBuildArgs: additionalBuildArgs,
             overrideFileNameVersion: overrideFileNameVersion,

pipelines/build/common/create_job_from_template.groovy

@@ -168,6 +168,7 @@ pipelineJob("$buildFolder/$JOB_NAME") {
                 <dt><strong>KEEP_TEST_REPORTDIR</strong></dt><dd>If true, test report dir (including core files where generated) will be kept even when the testcase passes, failed testcases always keep the report dir. Does not apply to JUnit jobs which are always kept, eg.openjdk.</dd>
                 <dt><strong>ACTIVE_NODE_TIMEOUT</strong></dt><dd>Number of minutes we will wait for a label-matching node to become active.</dd>
                 <dt><strong>CODEBUILD</strong></dt><dd>Use a dynamic codebuild machine if no other machine is available</dd>
+                <dt><strong>DEVKIT</strong></dt><dd>Use the DevKit with the specified url relative to [jenkinsDetails][devkitJobRoot], to compile with</dd>
                 <dt><strong>DOCKER_IMAGE</strong></dt><dd>Use a docker build environment</dd>
                 <dt><strong>DOCKER_ARGS</strong></dt><dd>Additional args to be used in conjuction with DOCKER_IMAGE</dd>
                 <dt><strong>DOCKER_FILE</strong></dt><dd>Relative path to a dockerfile to be built and used on top of the DOCKER_IMAGE</dd>

pipelines/build/common/openjdk_build_pipeline.groovy

@@ -12,6 +12,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+/* groovylint-disable MethodCount */
+
 import common.IndividualBuildConfig
 import common.MetaData
 import common.VersionInfo
@@ -1442,6 +1444,34 @@ class Build {
         return context.readFile(crossCompileVersionPath)
     }
 
+    /*
+     Download the given DevKit to ${WORKSPACE}/devkit
+     and return CONFIGURE_ARG "--with-devkit=${WORKSPACE}/devkit"
+     */
+    def downloadDevKit(devkit) {
+        def devkitJobRoot = Boolean.valueOf(buildConfig.USE_ADOPT_SHELL_SCRIPTS) ? ((String)ADOPT_DEFAULTS_JSON['jenkinsDetails']['devkitJobRoot']) : ((String)DEFAULTS_JSON['jenkinsDetails']['devkitJobRoot'])
+        def devkitUrl = devkitJobRoot + "/" + devkit
+        def devkitLoc = context.WORKSPACE + "/workspace/build/src/build/devkit"
+        context.println 'Downloading DevKit : ' + devkitUrl
+        context.println 'to location : ' + devkitLoc
+
+        context.withEnv(['devkitUrl='+devkitUrl, 'devkitLoc='+devkitLoc]) {
+            // groovylint-disable
+            context.sh '''
+                #!/bin/bash
+                set -eu
+                rm -rf "${devkitLoc}"
+                mkdir -p "${devkitLoc}"
+                cd "${devkitLoc}"
+                curl --fail --silent --show-error -o "devkit.tar.gz" "${devkitUrl}"
+                tar -xf "devkit.tar.gz"
+                rm "devkit.tar.gz"
+            '''
+            // groovylint-enable
+        }
+        return '--with-devkit=' + devkitLoc
+    }
+
     /*
      Display the current git repo information
      */
@@ -1539,6 +1569,14 @@ class Build {
                 // Use BUILD_REF override if specified
                 def adoptBranch = buildConfig.BUILD_REF ?: ADOPT_DEFAULTS_JSON['repository']['build_branch']
 
+                // Download devkit if specified
+                def devkit = ""
+                //if (buildConfig.DEVKIT != null && !buildConfig.DEVKIT.isEmpty()) {
+                    //devkit = downloadDevKit(buildConfig.DEVKIT)
+                    devkit = downloadDevKit("job/jdk21u/job/devkit_jdk21u_x64Linux/11/artifact/workspace/devkit-jdk21u-x86_64-linux-gnu.tar.gz")
+                //}
+context.println "Devkit arg=" + devkit
+
                 // Add platform config path so it can be used if the user doesn't have one
                 def splitAdoptUrl = ((String)ADOPT_DEFAULTS_JSON['repository']['build_url']) - ('.git').split('/')
                 // e.g. https://github.com/adoptium/temurin-build.git will produce adoptium/temurin-build
@@ -1566,7 +1604,8 @@ class Build {
                                     } else {
                                         signBuildArgs = '--make-exploded-image'
                                     }
-                                    context.withEnv(['BUILD_ARGS=' + signBuildArgs]) {
+                                    def signConfigureArgs = (devkit != "") ? env.CONFIGURE_ARGS + " " + devkit : env.CONFIGURE_ARGS
+                                    context.withEnv(['BUILD_ARGS=' + signBuildArgs, 'CONFIGURE_ARGS=' + signConfigureArgs]) {
                                         context.println 'Building an exploded image for signing'
                                         context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}")
                                     }
@@ -1679,12 +1718,16 @@ class Build {
                                     } else {
                                         assembleBuildArgs = '--assemble-exploded-image'
                                     }
-                                    context.withEnv(['BUILD_ARGS=' + assembleBuildArgs]) {
+                                    def assembleConfigureArgs = (devkit != "") ? env.CONFIGURE_ARGS + " " + devkit : env.CONFIGURE_ARGS
+                                    context.withEnv(['BUILD_ARGS=' + assembleBuildArgs, 'CONFIGURE_ARGS=' + assembleConfigureArgs]) {
                                         context.println 'Assembling the exploded image'
                                         context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}")
                                     }
                                 } else {
-                                    context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}")
+                                    def buildConfigureArgs = (devkit != "") ? env.CONFIGURE_ARGS + " " + devkit : env.CONFIGURE_ARGS
+                                    context.withEnv(['CONFIGURE_ARGS=' + buildConfigureArgs]) {
+                                        context.sh(script: "./${ADOPT_DEFAULTS_JSON['scriptDirectories']['buildfarm']}")
+                                    }
                                 }
                                 context.println '[CHECKOUT] Reverting pre-build adoptium/temurin-build checkout...'
                                 // Special case for the pr tester as checking out to the user's pipelines doesn't play nicely
@@ -1700,7 +1743,10 @@ class Build {
                                 repoHandler.setUserDefaultsJson(context, DEFAULTS_JSON)
                                 repoHandler.checkoutUserBuild(context)
                                 printGitRepoInfo()
-                                context.sh(script: "./${DEFAULTS_JSON['scriptDirectories']['buildfarm']}")
+                                def buildConfigureArgs = (devkit != "") ? env.CONFIGURE_ARGS + " " + devkit : env.CONFIGURE_ARGS
+                                context.withEnv(['CONFIGURE_ARGS=' + buildConfigureArgs]) {
+                                    context.sh(script: "./${DEFAULTS_JSON['scriptDirectories']['buildfarm']}")
+                                }
                                 context.println '[CHECKOUT] Reverting pre-build user temurin-build checkout...'
                                 repoHandler.checkoutUserPipelines(context)
                                 printGitRepoInfo()

pipelines/build/devkit/Tools.gmk.patch

@@ -0,0 +1,31 @@
+diff --git a/make/devkit/Tools.gmk b/make/devkit/Tools.gmk
+index 187320ca26e..63bfe418a10 100644
+--- a/make/devkit/Tools.gmk
++++ b/make/devkit/Tools.gmk
+@@ -62,6 +62,17 @@ ifeq ($(BASE_OS), OL)
+     BASE_URL := http://yum.oracle.com/repo/OracleLinux/OL6/4/base/$(ARCH)/
+     LINUX_VERSION := OL6.4
+   endif
++else ifeq ($(BASE_OS), Centos)
++  DEFAULT_OS_VERSION := 7
++  ifeq ($(BASE_OS_VERSION), )
++    BASE_OS_VERSION := $(DEFAULT_OS_VERSION)
++  endif
++  ifeq ($(ARCH), x86_64)
++    BASE_URL := https://vault.centos.org/$(BASE_OS_VERSION)/os/$(ARCH)/Packages/
++  else
++    BASE_URL := https://vault.centos.org/altarch/$(BASE_OS_VERSION)/os/$(ARCH)/Packages/
++  endif
++  LINUX_VERSION := Centos$(BASE_OS_VERSION)
+ else ifeq ($(BASE_OS), Fedora)
+   ifeq ($(ARCH), riscv64)
+     DEFAULT_OS_VERSION := rawhide/68692
+@@ -246,7 +257,7 @@ download-rpms:
+         # Only run this if rpm dir is empty.
+         ifeq ($(wildcard $(DOWNLOAD_RPMS)/*.rpm), )
+ 	  cd $(DOWNLOAD_RPMS) && \
+-	      wget -r -np -nd $(patsubst %, -A "*%*.rpm", $(RPM_LIST)) $(BASE_URL)
++	      wget -e robots=off -r -np -nd $(patsubst %, -A "*%*.rpm", $(RPM_LIST)) $(BASE_URL)
+         endif
+
+ ##########################################################################################

pipelines/build/devkit/binutils-2.39.patch

@@ -0,0 +1,38 @@
+diff --git a/gprofng/Makefile.am b/gprofng/Makefile.am
+index 2fcd695140ca6e3f06f3cc6844d498847a21a63d..96f8015700885e00b9e121cd61adede63c2b832e 100644 (file)
+--- a/gprofng/Makefile.am
++++ b/gprofng/Makefile.am
+@@ -23,11 +23,14 @@ AUTOMAKE_OPTIONS = dejagnu foreign
+ if BUILD_COLLECTOR
+     COLLECTOR_SUBDIRS = libcollector
+ endif
++if BUILD_MAN
++    DOC_SUBDIR = doc
++endif
+ if BUILD_SRC
+-    SRC_SUBDIRS = src gp-display-html doc
++    SRC_SUBDIRS = src gp-display-html $(DOC_SUBDIR)
+ endif
+ SUBDIRS = $(COLLECTOR_SUBDIRS) $(SRC_SUBDIRS)
+-DIST_SUBDIRS = libcollector src gp-display-html doc
++DIST_SUBDIRS = libcollector src gp-display-html $(DOC_SUBDIR)
+
+ # Setup the testing framework, if you have one
+ EXPECT = expect
+diff --git a/gprofng/Makefile.in b/gprofng/Makefile.in
+index fd5279b4df13ca21061bfc833797a0bb6ada62bc..d660f99233fedef9df9cae31629cd3ba57a51150 100644 (file)
+--- a/gprofng/Makefile.in
++++ b/gprofng/Makefile.in
+@@ -381,9 +381,10 @@ zlibinc = @zlibinc@
+ ACLOCAL_AMFLAGS = -I . -I .. 
+ AUTOMAKE_OPTIONS = dejagnu foreign
+ @BUILD_COLLECTOR_TRUE@COLLECTOR_SUBDIRS = libcollector
+-@BUILD_SRC_TRUE@SRC_SUBDIRS = src gp-display-html doc
++@BUILD_MAN_TRUE@DOC_SUBDIR = doc
++@BUILD_SRC_TRUE@SRC_SUBDIRS = src gp-display-html $(DOC_SUBDIR)
+ SUBDIRS = $(COLLECTOR_SUBDIRS) $(SRC_SUBDIRS)
+-DIST_SUBDIRS = libcollector src gp-display-html doc
++DIST_SUBDIRS = libcollector src gp-display-html $(DOC_SUBDIR)
+ RUNTEST = runtest
+ RUNTESTFLAGS = 
+ BASEDIR = $(srcdir)/..

pipelines/build/devkit/build_devkit.groovy

@@ -0,0 +1,105 @@
+package common
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+ * Builds a linux-gnu DevKit for the given openjdk version and architecture.
+ * Supported architectures:
+ *   x86_64
+ *   aarch64
+ *   ppc64le
+ *   s390x
+ */
+
+def build_devkit() {
+    stage('Build DevKit') {
+        def openjdkRepo = "https://github.com/openjdk/${params.VERSION}.git"
+
+        // Clone upstream openjdk repo
+        sh(script:"git clone --depth 1 ${openjdkRepo} ${params.VERSION}")
+
+        // Patch to support Centos7
+        sh(script:"cp pipelines/build/devkit/binutils-2.39.patch ${params.VERSION}/make/devkit/patches/${params.ARCH}-binutils-2.39.patch")
+        sh(script:"cd ${params.VERSION} && patch -p1<../pipelines/build/devkit/Tools.gmk.patch")
+
+        def devkit_target = "${params.ARCH}-linux-gnu"
+
+        // Perform devkit build
+        sh(script:"cd ${params.VERSION}/make/devkit && make TARGETS=${devkit_target} BASE_OS=${params.BASE_OS} BASE_OS_VERSION=${params.BASE_OS_VERSION}")
+
+        // Get gcc version and base OS from devkit.info
+        def gcc_ver=sh(script:'grep DEVKIT_NAME '+params.VERSION+'/build/devkit/result/'+devkit_target+'-to-'+devkit_target+'/devkit.info | cut -d"=" -f2 | tr -d "\\" \\n"', returnStdout: true)
+
+        def adoptium_devkit_release_tag = "${gcc_ver}-${devkit_target}-${params.BUILD}"
+
+        // Store Adoptium metadata within the devkit.info file
+        sh(script:"echo ADOPTIUM_DEVKIT_RELEASE_TAG=\"${adoptium_devkit_release_tag}\" >> ${params.VERSION}/build/devkit/result/${devkit_target}-to-${devkit_target}/devkit.info")
+
+        def devkit_file = "workspace/devkit-${adoptium_devkit_release_tag}.tar.gz"
+        println "devkit artifact filename = ${devkit_file}"
+
+        // Compress and archive
+        sh(script:"tar -cf - -C ${params.VERSION}/build/devkit/result/${devkit_target}-to-${devkit_target} . | GZIP=-9 gzip -c > ${devkit_file}")
+
+        // Create sha256.txt
+        sh(script:"sha256sum ${devkit_file} > ${devkit_file}.sha256.txt")
+
+        archiveArtifacts artifacts: "workspace/*"
+    }
+}
+
+def gpgSign() {
+    stage('GPG sign') {
+        def params = [
+                  string(name: 'UPSTREAM_JOB_NUMBER', value: "${env.BUILD_NUMBER}"),
+                  string(name: 'UPSTREAM_JOB_NAME',   value: "${env.JOB_NAME}"),
+                  string(name: 'UPSTREAM_DIR',        value: 'workspace')
+        ]
+
+        def signSHAsJob = build job: 'build-scripts/release/sign_temurin_gpg',
+               propagate: true,
+               parameters: params
+
+        copyArtifacts(
+               projectName: 'build-scripts/release/sign_temurin_gpg',
+               selector: specific("${signSHAsJob.getNumber()}"),
+               filter: '**/*.sig',
+               fingerprintArtifacts: true,
+               target: 'workspace',
+               flatten: true)
+
+        archiveArtifacts artifacts: "workspace/*.sig"
+    }
+}
+
+node(params.DEVKIT_BUILD_NODE) {
+  try {
+    cleanWs notFailBuild: true, disableDeferredWipeout: true, deleteDirs: true
+
+    docker.image(params.DOCKER_IMAGE).pull()
+    docker.image(params.DOCKER_IMAGE).inside() {
+        // Checkout pipelines code
+        checkout scm
+
+        // Create workspace for artifacts
+        sh("mkdir workspace")
+
+        build_devkit()
+        gpgSign()
+    }
+  } finally { 
+    cleanWs notFailBuild: true
+  } 
+}
+

pipelines/defaults.json

@@ -9,7 +9,8 @@
     },
     "jenkinsDetails"         : {
         "rootUrl"            : "https://ci.adoptium.net",
-        "rootDirectory"      : "build-scripts"
+        "rootDirectory"      : "build-scripts",
+        "devkitJobRoot"      : "https://ci.adoptium.net/job/build-scripts/job/utils/job/devkit"
     },
     "templateDirectories" : {
         "downstream"         : "pipelines/build/common/create_job_from_template.groovy",

pipelines/jobs/configurations/jdk21u_pipeline_config.groovy

@@ -21,6 +21,7 @@ class Config21 {
         x64Linux  : [
                 os                  : 'linux',
                 arch                : 'x64',
+                devkit              : 'job/jdk21u/job/devkit_jdk21u_x64Linux/10/artifact/workspace/devkit-jdk21u-x86_64-linux-gnu.tar.gz',
                 dockerImage         : 'adoptopenjdk/centos7_build_image',
                 dockerFile: [
                         openj9      : 'pipelines/build/dockerFiles/cuda.dockerfile'
@@ -112,6 +113,7 @@ class Config21 {
         ppc64leLinux    : [
                 os                  : 'linux',
                 arch                : 'ppc64le',
+                devkit              : 'job/jdk21u/job/devkit_jdk21u_ppc64leLinux/2/artifact/workspace/devkit-jdk21u-ppc64le-linux-gnu.tar.gz',
                 dockerImage         : 'adoptopenjdk/centos7_build_image',
                 test                : 'default',
                 reproducibleCompare : [
@@ -128,6 +130,7 @@ class Config21 {
         aarch64Linux    : [
                 os                  : 'linux',
                 arch                : 'aarch64',
+                devkit              : 'job/jdk21u/job/devkit_jdk21u_aarch64Linux/18/artifact/workspace/devkit-jdk21u-aarch64-linux-gnu.tar.gz',
                 dockerImage         : 'adoptopenjdk/centos7_build_image',
                 test                : 'default',
                 reproducibleCompare : [